def handle_regular_connect(self, f): self.connect_request = True try: self.set_server((f.request.host, f.request.port)) except (exceptions.ProtocolException, exceptions.NetlibException) as e: # HTTPS tasting means that ordinary errors like resolution # and connection errors can happen here. self.send_error_response(502, repr(e)) f.error = flow.Error(str(e)) self.channel.ask("error", f) return False if f.response: resp = f.response else: resp = http.make_connect_response(f.request.data.http_version) self.send_response(resp) if is_ok(resp.status_code): layer = self.ctx.next_layer(self) layer() return False
def check_killed(self, emit_error_hook: bool) -> layer.CommandGenerator[bool]: killed_by_us = ( self.flow.error and self.flow.error.msg == flow.Error.KILLED_MESSAGE ) # The client may have closed the connection while we were waiting for the hook to complete. # We peek into the event queue to see if that is the case. killed_by_remote = None for evt in self._paused_event_queue: if isinstance(evt, RequestProtocolError): killed_by_remote = evt.message break if killed_by_remote: if not self.flow.error: self.flow.error = flow.Error(killed_by_remote) if killed_by_us or killed_by_remote: if emit_error_hook: yield HttpErrorHook(self.flow) # Use the special NO_RESPONSE status code to make sure that no error message is sent to the client. yield SendHttp( ResponseProtocolError(self.stream_id, "killed", status_codes.NO_RESPONSE), self.context.client ) self._handle_event = self.state_errored return True return False
def handle_protocol_error( self, event: Union[RequestProtocolError, ResponseProtocolError] ) -> layer.CommandGenerator[None]: is_client_error_but_we_already_talk_upstream = ( isinstance(event, RequestProtocolError) and self.client_state in (self.state_stream_request_body, self.state_done) and self.server_state != self.state_errored ) need_error_hook = not ( self.client_state in (self.state_wait_for_request_headers, self.state_errored) or self.server_state in (self.state_done, self.state_errored) ) if is_client_error_but_we_already_talk_upstream: yield SendHttp(event, self.context.server) self.client_state = self.state_errored if need_error_hook: # We don't want to trigger both a response hook and an error hook, # so we need to check if the response is done yet or not. self.flow.error = flow.Error(event.message) yield HttpErrorHook(self.flow) if (yield from self.check_killed(False)): return if isinstance(event, ResponseProtocolError): if self.client_state != self.state_errored: yield SendHttp(event, self.context.client) self.server_state = self.state_errored
def check_killed(self, emit_error_hook: bool) -> layer.CommandGenerator[bool]: killed_by_us = (self.flow.error and self.flow.error.msg == flow.Error.KILLED_MESSAGE) # The client may have closed the connection while we were waiting for the hook to complete. # We peek into the event queue to see if that is the case. killed_by_remote = None for evt in self._paused_event_queue: if isinstance(evt, RequestProtocolError): killed_by_remote = evt.message break if killed_by_remote: if not self.flow.error: self.flow.error = flow.Error(killed_by_remote) if killed_by_us or killed_by_remote: if emit_error_hook: yield HttpErrorHook(self.flow) # For HTTP/2 we only want to kill the specific stream, for HTTP/1 we want to kill the connection # *without* sending an HTTP response (that could be achieved by the user by setting flow.response). if self.context.client.alpn == b"h2": yield SendHttp(ResponseProtocolError(self.stream_id, "killed"), self.context.client) else: if self.context.client.state & ConnectionState.CAN_WRITE: yield commands.CloseConnection(self.context.client) self._handle_event = self.state_errored return True return False
def __call__(self): self.connect() if not self.ignore: f = tcp.TCPFlow(self.client_conn, self.server_conn, self) self.channel.ask("tcp_start", f) buf = memoryview(bytearray(self.chunk_size)) client = self.client_conn.connection server = self.server_conn.connection conns = [client, server] # https://github.com/openssl/openssl/issues/6234 for conn in conns: if isinstance(conn, SSL.Connection) and hasattr( SSL._lib, "SSL_clear_mode"): SSL._lib.SSL_clear_mode(conn._ssl, SSL._lib.SSL_MODE_AUTO_RETRY) try: while not self.channel.should_exit.is_set(): r = mitmproxy.net.tcp.ssl_read_select(conns, 10) for conn in r: dst = server if conn == client else client try: size = conn.recv_into(buf, self.chunk_size) except (SSL.WantReadError, SSL.WantWriteError): continue if not size: conns.remove(conn) # Shutdown connection to the other peer if isinstance(conn, SSL.Connection): # We can't half-close a connection, so we just close everything here. # Sockets will be cleaned up on a higher level. return else: dst.shutdown(socket.SHUT_WR) if len(conns) == 0: return continue tcp_message = tcp.TCPMessage(dst == server, buf[:size].tobytes()) if not self.ignore: f.messages.append(tcp_message) self.channel.ask("tcp_message", f) dst.sendall(tcp_message.content) except (OSError, exceptions.TcpException, SSL.Error) as e: if not self.ignore: f.error = flow.Error( "TCP connection closed unexpectedly: {}".format(repr(e))) self.channel.tell("tcp_error", f) finally: if not self.ignore: self.channel.tell("tcp_end", f)
def test_getset_state(self): f = tutils.tflow(resp=True) state = f.get_state() assert f.get_state() == http.HTTPFlow.from_state(state).get_state() f.response = None f.error = flow.Error("error") state = f.get_state() assert f.get_state() == http.HTTPFlow.from_state(state).get_state() f2 = f.copy() f2.id = f.id # copy creates a different uuid assert f.get_state() == f2.get_state() assert not f == f2 f2.error = flow.Error("e2") assert not f == f2 f.set_state(f2.get_state()) assert f.get_state() == f2.get_state()
def test_getset_state(self): f = tflow(resp=True) state = f.get_state() assert f.get_state() == HTTPFlow.from_state(state).get_state() f.response = None f.error = flow.Error("error") state = f.get_state() assert f.get_state() == HTTPFlow.from_state(state).get_state() f2 = f.copy() f2.id = f.id # copy creates a different uuid assert f.get_state() == f2.get_state() assert not f == f2 f2.error = flow.Error("e2") assert not f == f2 f2.backup() f2.intercept() # to change the state f.set_state(f2.get_state()) assert f.get_state() == f2.get_state()
def __call__(self): self.flow = WebSocketFlow(self.client_conn, self.server_conn, self.handshake_flow) self.flow.metadata['websocket_handshake'] = self.handshake_flow.id self.handshake_flow.metadata['websocket_flow'] = self.flow.id self.channel.ask("websocket_start", self.flow) conns = [c.connection for c in self.connections.keys()] close_received = False try: while not self.channel.should_exit.is_set(): self._inject_messages(self.client_conn, self.flow._inject_messages_client) self._inject_messages(self.server_conn, self.flow._inject_messages_server) r = tcp.ssl_read_select(conns, 0.1) for conn in r: source_conn = self.client_conn if conn == self.client_conn.connection else self.server_conn other_conn = self.server_conn if conn == self.client_conn.connection else self.client_conn is_server = (source_conn == self.server_conn) header, frame, consumed_bytes = websocket.read_frame( source_conn.rfile) self.log( "WebSocket Frame from {}: {}, {}".format( "server" if is_server else "client", header, frame, ), "debug") data = self.connections[source_conn].receive_data( consumed_bytes) source_conn.send(data) if close_received: return for event in self.connections[source_conn].events(): if not self._handle_event(event, source_conn, other_conn, is_server): if not close_received: close_received = True except (socket.error, exceptions.TcpException, SSL.Error) as e: s = 'server' if is_server else 'client' self.flow.error = flow.Error( "WebSocket connection closed unexpectedly by {}: {}".format( s, repr(e))) self.channel.tell("websocket_error", self.flow) finally: self.flow.ended = True self.channel.tell("websocket_end", self.flow)
def websocket_end(self, f: http.HTTPFlow): assert f.websocket is not None # satisfy type checker if self.match(f): if f.websocket.close_code in {1000, 1001, 1005}: c = 'client' if f.websocket.closed_by_client else 'server' self.echo(f"WebSocket connection closed by {c}: {f.websocket.close_code} {f.websocket.close_reason}") else: error = flow.Error(f"WebSocket Error: {self.format_websocket_error(f.websocket)}") self.echo( f"Error in WebSocket connection to {human.format_address(f.server_conn.address)}: {error}", fg="red" )
def start(self, _) -> layer.CommandGenerator[None]: if self.flow: yield TcpStartHook(self.flow) if not self.context.server.connected: err = yield commands.OpenConnection(self.context.server) if err: if self.flow: self.flow.error = flow.Error(str(err)) yield TcpErrorHook(self.flow) yield commands.CloseConnection(self.context.client) self._handle_event = self.done return self._handle_event = self.relay_messages
def test_err(self): c = state.State() f = tutils.tflow() c.add_flow(f) f.error = flow.Error("message") assert c.update_flow(f) c = state.State() f = tutils.tflow() c.add_flow(f) c.set_view_filter("~e") assert not c.view f.error = tutils.terr() assert c.update_flow(f) assert c.view
def __call__(self): self.connect() if not self.ignore: f = udp.UDPFlow(self.client_conn, self.server_conn, self) self.channel.ask("udp_start", f) buf = memoryview(bytearray(self.chunk_size)) client = self.client_conn.connection server = self.server_conn.connection conns = [client, server] # https://github.com/openssl/openssl/issues/6234 #for conn in conns: # if isinstance(conn, SSL.Connection) and hasattr(SSL._lib, "SSL_clear_mode"): # SSL._lib.SSL_clear_mode(conn._ssl, SSL._lib.SSL_MODE_AUTO_RETRY) try: while not self.channel.should_exit.is_set(): r = mitmproxy.net.udp.read_select(conns, 10) for conn in r: dst = server if conn == client else client size = conn.recv_into(buf, self.chunk_size) if not size: conns.remove(conn) # Shutdown connection to the other peer dst.shutdown(socket.SHUT_WR) if len(conns) == 0: return continue udp_message = udp.UDPMessage(dst == server, buf[:size].tobytes()) if not self.ignore: f.messages.append(udp_message) self.channel.ask("udp_message", f) dst.sendall(udp_message.content) except (OSError, exceptions.UdpException) as e: if not self.ignore: f.error = flow.Error( "UDP connection closed unexpectedly: {}".format(repr(e))) self.channel.tell("udp_error", f) finally: if not self.ignore: self.channel.tell("udp_end", f)
async def test_all(self): opts = options.Options( mode="reverse:https://use-this-domain" ) s = State() with taddons.context(s, options=opts) as ctx: f = tflow.tflow(req=None) await ctx.master.addons.handle_lifecycle(server_hooks.ClientConnectedHook(f.client_conn)) f.request = mitmproxy.test.tutils.treq() await ctx.master.addons.handle_lifecycle(layers.http.HttpRequestHook(f)) assert len(s.flows) == 1 f.response = mitmproxy.test.tutils.tresp() await ctx.master.addons.handle_lifecycle(layers.http.HttpResponseHook(f)) assert len(s.flows) == 1 await ctx.master.addons.handle_lifecycle(server_hooks.ClientDisconnectedHook(f.client_conn)) f.error = flow.Error("msg") await ctx.master.addons.handle_lifecycle(layers.http.HttpErrorHook(f))
def test_all(self): s = tservers.TestState() fm = master.Master(None, DummyServer()) fm.addons.add(s) f = tflow.tflow(req=None) fm.clientconnect(f.client_conn) f.request = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq()) fm.request(f) assert s.flow_count() == 1 f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) fm.response(f) assert s.flow_count() == 1 fm.clientdisconnect(f.client_conn) f.error = flow.Error("msg") fm.error(f) fm.shutdown()
def test_all(self): s = tservers.TestState() fm = master.Master(None, DummyServer()) fm.addons.add(s) f = tflow.tflow(req=None) fm.addons.handle_lifecycle("clientconnect", f.client_conn) f.request = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq()) fm.addons.handle_lifecycle("request", f) assert len(s.flows) == 1 f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) fm.addons.handle_lifecycle("response", f) assert len(s.flows) == 1 fm.addons.handle_lifecycle("clientdisconnect", f.client_conn) f.error = flow.Error("msg") fm.addons.handle_lifecycle("error", f) fm.shutdown()
async def test_all(self): opts = options.Options( mode="reverse:https://use-this-domain" ) s = tservers.TestState() with taddons.context(s, options=opts) as ctx: f = tflow.tflow(req=None) await ctx.master.addons.handle_lifecycle("clientconnect", f.client_conn) f.request = mitmproxy.test.tutils.treq() await ctx.master.addons.handle_lifecycle("request", f) assert len(s.flows) == 1 f.response = mitmproxy.test.tutils.tresp() await ctx.master.addons.handle_lifecycle("response", f) assert len(s.flows) == 1 await ctx.master.addons.handle_lifecycle("clientdisconnect", f.client_conn) f.error = flow.Error("msg") await ctx.master.addons.handle_lifecycle("error", f)
def __call__(self): self.flow = WebSocketFlow(self.client_conn, self.server_conn, self.handshake_flow, self) self.flow.metadata['websocket_handshake'] = self.handshake_flow.id self.handshake_flow.metadata['websocket_flow'] = self.flow.id self.channel.ask("websocket_start", self.flow) conns = [c.connection for c in self.connections.keys()] close_received = False try: while not self.channel.should_exit.is_set(): r = tcp.ssl_read_select(conns, 0.1) for conn in r: source_conn = self.client_conn if conn == self.client_conn.connection else self.server_conn other_conn = self.server_conn if conn == self.client_conn.connection else self.client_conn is_server = (source_conn == self.server_conn) frame = websockets.Frame.from_file(source_conn.rfile) self.connections[source_conn].receive_bytes(bytes(frame)) source_conn.send( self.connections[source_conn].bytes_to_send()) if close_received: return for event in self.connections[source_conn].events(): if not self._handle_event(event, source_conn, other_conn, is_server): if not close_received: close_received = True except (socket.error, exceptions.TcpException, SSL.Error) as e: s = 'server' if is_server else 'client' self.flow.error = flow.Error( "WebSocket connection closed unexpectedly by {}: {}".format( s, repr(e))) self.channel.tell("websocket_error", self.flow) finally: self.channel.tell("websocket_end", self.flow)
def test_all(self): s = tservers.TestState() fm = master.Master(None) fm.addons.add(s) f = tflow.tflow(req=None) fm.addons.handle_lifecycle("clientconnect", f.client_conn) f.request = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq()) fm.addons.handle_lifecycle("request", f) assert len(s.flows) == 1 f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) fm.addons.handle_lifecycle("response", f) assert len(s.flows) == 1 fm.addons.handle_lifecycle("clientdisconnect", f.client_conn) f.error = flow.Error("msg") fm.addons.handle_lifecycle("error", f) fm.tell("foo", f) with pytest.raises(ControlException): fm.tick(timeout=1) fm.shutdown()
def _process_flow(self, f): try: try: request = self.read_request_headers(f) except exceptions.HttpReadDisconnect: # don't throw an error for disconnects that happen # before/between requests. return False f.request = request if request.first_line_format == "authority": # The standards are silent on what we should do with a CONNECT # request body, so although it's not common, it's allowed. f.request.data.content = b"".join( self.read_request_body(f.request) ) f.request.timestamp_end = time.time() self.channel.ask("http_connect", f) if self.mode is HTTPMode.regular: return self.handle_regular_connect(f) elif self.mode is HTTPMode.upstream: return self.handle_upstream_connect(f) else: msg = "Unexpected CONNECT request." self.send_error_response(400, msg) raise exceptions.ProtocolException(msg) self.channel.ask("requestheaders", f) if request.headers.get("expect", "").lower() == "100-continue": # TODO: We may have to use send_response_headers for HTTP2 # here. self.send_response(http.expect_continue_response) request.headers.pop("expect") request.data.content = b"".join(self.read_request_body(request)) request.timestamp_end = time.time() validate_request_form(self.mode, request) except exceptions.HttpException as e: # We optimistically guess there might be an HTTP client on the # other end self.send_error_response(400, repr(e)) raise exceptions.ProtocolException( "HTTP protocol error in client request: {}".format(e) ) self.log("request", "debug", [repr(request)]) # set first line format to relative in regular mode, # see https://github.com/mitmproxy/mitmproxy/issues/1759 if self.mode is HTTPMode.regular and request.first_line_format == "absolute": request.first_line_format = "relative" # update host header in reverse proxy mode if self.config.options.mode == "reverse": f.request.headers["Host"] = self.config.upstream_server.address.host # Determine .scheme, .host and .port attributes for inline scripts. For # absolute-form requests, they are directly given in the request. For # authority-form requests, we only need to determine the request # scheme. For relative-form requests, we need to determine host and # port as well. if self.mode is HTTPMode.transparent: # Setting request.host also updates the host header, which we want # to preserve host_header = f.request.headers.get("host", None) f.request.host = self.__initial_server_conn.address.host f.request.port = self.__initial_server_conn.address.port if host_header: f.request.headers["host"] = host_header f.request.scheme = "https" if self.__initial_server_tls else "http" self.channel.ask("request", f) try: if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers): # We only support RFC6455 with WebSockets version 13 # allow inline scripts to manipulate the client handshake self.channel.ask("websocket_handshake", f) if not f.response: self.establish_server_connection( f.request.host, f.request.port, f.request.scheme ) def get_response(): self.send_request(f.request) f.response = self.read_response_headers() try: get_response() except exceptions.NetlibException as e: self.log( "server communication error: %s" % repr(e), level="debug" ) # In any case, we try to reconnect at least once. This is # necessary because it might be possible that we already # initiated an upstream connection after clientconnect that # has already been expired, e.g consider the following event # log: # > clientconnect (transparent mode destination known) # > serverconnect (required for client tls handshake) # > read n% of large request # > server detects timeout, disconnects # > read (100-n)% of large request # > send large request upstream if isinstance(e, exceptions.Http2ProtocolException): # do not try to reconnect for HTTP2 raise exceptions.ProtocolException( "First and only attempt to get response via HTTP2 failed." ) self.disconnect() self.connect() get_response() # call the appropriate script hook - this is an opportunity for # an inline script to set f.stream = True self.channel.ask("responseheaders", f) if f.response.stream: f.response.data.content = None else: f.response.data.content = b"".join( self.read_response_body(f.request, f.response) ) f.response.timestamp_end = time.time() # no further manipulation of self.server_conn beyond this point # we can safely set it as the final attribute value here. f.server_conn = self.server_conn else: # response was set by an inline script. # we now need to emulate the responseheaders hook. self.channel.ask("responseheaders", f) self.log("response", "debug", [repr(f.response)]) self.channel.ask("response", f) if not f.response.stream: # no streaming: # we already received the full response from the server and can # send it to the client straight away. self.send_response(f.response) else: # streaming: # First send the headers and then transfer the response incrementally self.send_response_headers(f.response) chunks = self.read_response_body( f.request, f.response ) if callable(f.response.stream): chunks = f.response.stream(chunks) self.send_response_body(f.response, chunks) f.response.timestamp_end = time.time() if self.check_close_connection(f): return False # Handle 101 Switching Protocols if f.response.status_code == 101: # Handle a successful HTTP 101 Switching Protocols Response, # received after e.g. a WebSocket upgrade request. # Check for WebSockets handshake is_websockets = ( websockets.check_handshake(f.request.headers) and websockets.check_handshake(f.response.headers) ) if is_websockets and not self.config.options.websockets: self.log( "Client requested WebSocket connection, but the protocol is disabled.", "info" ) if is_websockets and self.config.options.websockets: layer = pwebsockets.WebSocketsLayer(self, f) else: layer = self.ctx.next_layer(self) layer() return False # should never be reached except (exceptions.ProtocolException, exceptions.NetlibException) as e: self.send_error_response(502, repr(e)) if not f.response: f.error = flow.Error(str(e)) self.channel.ask("error", f) return False else: raise exceptions.ProtocolException( "Error in HTTP connection: %s" % repr(e) ) finally: if f: f.live = False return True
def replay(self, f): # pragma: no cover f.live = True r = f.request bsl = human.parse_size(self.options.body_size_limit) first_line_format_backup = r.first_line_format server = None global new, cur_cycle, cur_group try: f.response = None # If we have a channel, run script hooks. request_reply = self.channel.ask("request", f) if isinstance(request_reply, http.HTTPResponse): f.response = request_reply if not f.response: # In all modes, we directly connect to the server displayed if self.options.mode.startswith("upstream:"): server_address = server_spec.parse_with_mode( self.options.mode)[1].address server = connections.ServerConnection(server_address) server.connect() if r.scheme == "https": connect_request = http.make_connect_request( (r.data.host, r.port)) server.wfile.write( http1.assemble_request(connect_request)) server.wfile.flush() resp = http1.read_response(server.rfile, connect_request, body_size_limit=bsl) if resp.status_code != 200: raise exceptions.ReplayException( "Upstream server refuses CONNECT request") server.establish_tls( sni=f.server_conn.sni, **tls.client_arguments_from_options(self.options)) r.first_line_format = "relative" else: r.first_line_format = "absolute" else: server_address = (r.host, r.port) server = connections.ServerConnection(server_address) server.connect() if r.scheme == "https": server.establish_tls( sni=f.server_conn.sni, **tls.client_arguments_from_options(self.options)) r.first_line_format = "relative" server.wfile.write(http1.assemble_request(r)) server.wfile.flush() if f.server_conn: f.server_conn.close() f.server_conn = server f.response = http.HTTPResponse.wrap( http1.read_response(server.rfile, r, body_size_limit=bsl)) response_reply = self.channel.ask("response", f) #new.append(f) #record the response cur_cycle[cur_group] = f if response_reply == exceptions.Kill: raise exceptions.Kill() except (exceptions.ReplayException, exceptions.NetlibException) as e: f.error = flow.Error(str(e)) self.channel.ask("error", f) except exceptions.Kill: self.channel.tell("log", log.LogEntry("Connection killed", "info")) except Exception as e: self.channel.tell("log", log.LogEntry(repr(e), "error")) finally: r.first_line_format = first_line_format_backup f.live = False if server.connected(): server.finish() server.close()
def _load_http_error(o: http_pb2.HTTPError) -> typing.Optional[flow.Error]: d = {} for m in ['msg', 'timestamp']: if hasattr(o, m) and getattr(o, m): d[m] = getattr(o, m) return None if not d else flow.Error(**d)
def __call__(self): if self.mode == "transparent": self.__initial_server_tls = self.server_tls self.__initial_server_conn = self.server_conn while True: f = http.HTTPFlow(self.client_conn, self.server_conn, live=self) try: request = self.get_request_from_client(f) # Make sure that the incoming request matches our expectations self.validate_request(request) except exceptions.HttpReadDisconnect: # don't throw an error for disconnects that happen before/between requests. return except exceptions.HttpException as e: # We optimistically guess there might be an HTTP client on the # other end self.send_error_response(400, repr(e)) raise exceptions.ProtocolException( "HTTP protocol error in client request: {}".format(e)) self.log("request", "debug", [repr(request)]) # Handle Proxy Authentication # Proxy Authentication conceptually does not work in transparent mode. # We catch this misconfiguration on startup. Here, we sort out requests # after a successful CONNECT request (which do not need to be validated anymore) if not (self.http_authenticated or self.authenticate(request)): return f.request = request try: # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.first_line_format == "authority": self.handle_regular_mode_connect(request) return except (exceptions.ProtocolException, exceptions.NetlibException) as e: # HTTPS tasting means that ordinary errors like resolution and # connection errors can happen here. self.send_error_response(502, repr(e)) f.error = flow.Error(str(e)) self.channel.ask("error", f) return # update host header in reverse proxy mode if self.config.options.mode == "reverse": f.request.headers[ "Host"] = self.config.upstream_server.address.host # set upstream auth if self.mode == "upstream" and self.config.upstream_auth is not None: f.request.headers[ "Proxy-Authorization"] = self.config.upstream_auth self.process_request_hook(f) try: if websockets.check_handshake( request.headers) and websockets.check_client_version( request.headers): # We only support RFC6455 with WebSockets version 13 # allow inline scripts to manipulate the client handshake self.channel.ask("websocket_handshake", f) if not f.response: self.establish_server_connection(f.request.host, f.request.port, f.request.scheme) self.get_response_from_server(f) else: # response was set by an inline script. # we now need to emulate the responseheaders hook. self.channel.ask("responseheaders", f) self.log("response", "debug", [repr(f.response)]) self.channel.ask("response", f) self.send_response_to_client(f) if self.check_close_connection(f): return # Handle 101 Switching Protocols if f.response.status_code == 101: return self.handle_101_switching_protocols(f) # Upstream Proxy Mode: Handle CONNECT if f.request.first_line_format == "authority" and f.response.status_code == 200: self.handle_upstream_mode_connect(f.request.copy()) return except (exceptions.ProtocolException, exceptions.NetlibException) as e: self.send_error_response(502, repr(e)) if not f.response: f.error = flow.Error(str(e)) self.channel.ask("error", f) return else: raise exceptions.ProtocolException( "Error in HTTP connection: %s" % repr(e)) finally: if f: f.live = False
def test_repr(self): e = flow.Error("yay") assert repr(e) assert str(e)
def terr(content="error"): """ @return: mitmproxy.proxy.protocol.primitives.Error """ err = flow.Error(content) return err
def check_body_size(self, request: bool) -> layer.CommandGenerator[bool]: """ Check if the body size exceeds limits imposed by stream_large_bodies or body_size_limit. Returns `True` if the body size exceeds body_size_limit and further processing should be stopped. """ if not (self.context.options.stream_large_bodies or self.context.options.body_size_limit): return False # Step 1: Determine the expected body size. This can either come from a known content-length header, # or from the amount of currently buffered bytes (e.g. for chunked encoding). response = not request expected_size: Optional[int] # the 'late' case: we already started consuming the body if request and self.request_body_buf: expected_size = len(self.request_body_buf) elif response and self.response_body_buf: expected_size = len(self.response_body_buf) else: # the 'early' case: we have not started consuming the body try: expected_size = expected_http_body_size( self.flow.request, self.flow.response if response else None) except ValueError: # pragma: no cover # we just don't stream/kill malformed content-length headers. expected_size = None if expected_size is None or expected_size <= 0: return False # Step 2: Do we need to abort this? max_total_size = human.parse_size(self.context.options.body_size_limit) if max_total_size is not None and expected_size > max_total_size: if request and not self.request_body_buf: yield HttpRequestHeadersHook(self.flow) if response and not self.response_body_buf: yield HttpResponseHeadersHook(self.flow) err_msg = f"{'Request' if request else 'Response'} body exceeds mitmproxy's body_size_limit." err_code = 413 if request else 502 self.flow.error = flow.Error(err_msg) yield HttpErrorHook(self.flow) yield SendHttp( ResponseProtocolError(self.stream_id, err_msg, err_code), self.context.client) self.client_state = self.state_errored if response: yield SendHttp( RequestProtocolError(self.stream_id, err_msg, err_code), self.context.server) self.server_state = self.state_errored self.flow.live = False return True # Step 3: Do we need to stream this? max_stream_size = human.parse_size( self.context.options.stream_large_bodies) if max_stream_size is not None and expected_size > max_stream_size: if request: self.flow.request.stream = True if self.request_body_buf: # clear buffer and then fake a DataReceived event with everything we had in the buffer so far. body_buf = self.request_body_buf self.request_body_buf = b"" yield from self.start_request_stream() yield from self.handle_event( RequestData(self.stream_id, body_buf)) if response: assert self.flow.response self.flow.response.stream = True if self.response_body_buf: body_buf = self.response_body_buf self.response_body_buf = b"" yield from self.start_response_stream() yield from self.handle_event( ResponseData(self.stream_id, body_buf)) return False
def terr(content: str = "error") -> flow.Error: err = flow.Error(content, 946681207) return err
def run(self): r = self.f.request first_line_format_backup = r.first_line_format server = None try: self.f.response = None # If we have a channel, run script hooks. if self.channel: request_reply = self.channel.ask("request", self.f) if isinstance(request_reply, http.HTTPResponse): self.f.response = request_reply if not self.f.response: # In all modes, we directly connect to the server displayed if self.config.options.mode == "upstream": server_address = self.config.upstream_server.address server = connections.ServerConnection(server_address, (self.config.options.listen_host, 0)) server.connect() if r.scheme == "https": connect_request = http.make_connect_request((r.data.host, r.port)) server.wfile.write(http1.assemble_request(connect_request)) server.wfile.flush() resp = http1.read_response( server.rfile, connect_request, body_size_limit=self.config.options.body_size_limit ) if resp.status_code != 200: raise exceptions.ReplayException("Upstream server refuses CONNECT request") server.establish_ssl( self.config.clientcerts, sni=self.f.server_conn.sni ) r.first_line_format = "relative" else: r.first_line_format = "absolute" else: server_address = (r.host, r.port) server = connections.ServerConnection( server_address, (self.config.options.listen_host, 0) ) server.connect() if r.scheme == "https": server.establish_ssl( self.config.clientcerts, sni=self.f.server_conn.sni ) r.first_line_format = "relative" server.wfile.write(http1.assemble_request(r)) server.wfile.flush() self.f.server_conn = server self.f.response = http.HTTPResponse.wrap( http1.read_response( server.rfile, r, body_size_limit=self.config.options.body_size_limit ) ) if self.channel: response_reply = self.channel.ask("response", self.f) if response_reply == exceptions.Kill: raise exceptions.Kill() except (exceptions.ReplayException, exceptions.NetlibException) as e: self.f.error = flow.Error(str(e)) if self.channel: self.channel.ask("error", self.f) except exceptions.Kill: # Kill should only be raised if there's a channel in the # first place. self.channel.tell( "log", log.LogEntry("Connection killed", "info") ) except Exception: self.channel.tell( "log", log.LogEntry(traceback.format_exc(), "error") ) finally: r.first_line_format = first_line_format_backup self.f.live = False if server.connected(): server.finish()
def relay_messages( self, event: events.ConnectionEvent) -> layer.CommandGenerator[None]: from_client = event.connection == self.context.client from_str = 'client' if from_client else 'server' if from_client: src_ws = self.client_ws dst_ws = self.server_ws else: src_ws = self.server_ws dst_ws = self.client_ws if isinstance(event, events.DataReceived): src_ws.receive_data(event.data) elif isinstance(event, events.ConnectionClosed): src_ws.receive_data(None) else: # pragma: no cover raise AssertionError(f"Unexpected event: {event}") for ws_event in src_ws.events(): if isinstance(ws_event, wsproto.events.Message): src_ws.frame_buf.append(ws_event.data) if ws_event.message_finished: if isinstance(ws_event, wsproto.events.TextMessage): frame_type = Opcode.TEXT content = "".join(src_ws.frame_buf) # type: ignore else: frame_type = Opcode.BINARY content = b"".join(src_ws.frame_buf) # type: ignore fragmentizer = Fragmentizer(src_ws.frame_buf) src_ws.frame_buf.clear() message = websocket.WebSocketMessage( frame_type, from_client, content) self.flow.messages.append(message) yield WebsocketMessageHook(self.flow) assert not message.killed # this is deprecated, instead we should have .content set to emptystr. for msg in fragmentizer(message.content): yield dst_ws.send2(msg) elif isinstance(ws_event, (wsproto.events.Ping, wsproto.events.Pong)): yield commands.Log( f"Received WebSocket {ws_event.__class__.__name__.lower()} from {from_str} " f"(payload: {bytes(ws_event.payload)!r})") yield dst_ws.send2(ws_event) elif isinstance(ws_event, wsproto.events.CloseConnection): self.flow.close_sender = from_str self.flow.close_code = ws_event.code self.flow.close_reason = ws_event.reason for ws in [self.server_ws, self.client_ws]: if ws.state in { ConnectionState.OPEN, ConnectionState.REMOTE_CLOSING }: # response == original event, so no need to differentiate here. yield ws.send2(ws_event) yield commands.CloseConnection(ws.conn) if ws_event.code in {1000, 1001, 1005}: yield WebsocketEndHook(self.flow) else: self.flow.error = flow.Error( f"WebSocket Error: {format_close_event(ws_event)}") yield WebsocketErrorHook(self.flow) self._handle_event = self.done else: # pragma: no cover raise AssertionError(f"Unexpected WebSocket event: {ws_event}")
def replay(self, f): # pragma: no cover f.live = True r = f.request bsl = human.parse_size(self.options.body_size_limit) authority_backup = r.authority server = None try: f.response = None # If we have a channel, run script hooks. request_reply = self.channel.ask("request", f) if isinstance(request_reply, http.HTTPResponse): f.response = request_reply if not f.response: # In all modes, we directly connect to the server displayed if self.options.mode.startswith("upstream:"): server_address = server_spec.parse_with_mode( self.options.mode)[1].address server = connections.ServerConnection(server_address) server.connect() if r.scheme == "https": connect_request = http.make_connect_request( (r.data.host, r.port)) server.wfile.write( http1.assemble_request(connect_request)) server.wfile.flush() resp = http1.read_response(server.rfile, connect_request, body_size_limit=bsl) if resp.status_code != 200: raise exceptions.ReplayException( "Upstream server refuses CONNECT request") server.establish_tls( sni=f.server_conn.sni, **tls.client_arguments_from_options(self.options)) r.authority = b"" else: r.authority = hostport(r.scheme, r.host, r.port) else: server_address = (r.host, r.port) server = connections.ServerConnection(server_address) server.connect() if r.scheme == "https": server.establish_tls( sni=f.server_conn.sni, **tls.client_arguments_from_options(self.options)) r.authority = "" server.wfile.write(http1.assemble_request(r)) server.wfile.flush() r.timestamp_start = r.timestamp_end = time.time() if f.server_conn: f.server_conn.close() f.server_conn = server f.response = http1.read_response(server.rfile, r, body_size_limit=bsl) response_reply = self.channel.ask("response", f) if response_reply == exceptions.Kill: raise exceptions.Kill() except (exceptions.ReplayException, exceptions.NetlibException) as e: f.error = flow.Error(str(e)) self.channel.ask("error", f) except exceptions.Kill: self.channel.tell("log", log.LogEntry(flow.Error.KILLED_MESSAGE, "info")) except Exception as e: self.channel.tell("log", log.LogEntry(repr(e), "error")) finally: r.authority = authority_backup f.live = False if server and server.connected(): server.finish() server.close()
def kill(self, force=False): # pragma: no cover warnings.warn( "reply.kill() is deprecated, use flow.kill() or set the error attribute instead.", DeprecationWarning, stacklevel=2) self.obj.error = flow.Error(flow.Error.KILLED_MESSAGE)