def test_stream_chunked(self): connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) connection.connect(("127.0.0.1", self.proxy.port)) fconn = connection.makefile() spec = '200:h"Transfer-Encoding"="chunked":r:b"4\\r\\nthis\\r\\n7\\r\\nisatest\\r\\n0\\r\\n\\r\\n"' connection.send( "GET %s/p/%s HTTP/1.1\r\n" % (self.server.urlbase, spec)) connection.send("\r\n") httpversion, code, msg, headers, content = http.read_response( fconn, "GET", None, include_body=False) assert headers["Transfer-Encoding"][0] == 'chunked' assert code == 200 chunks = list( content for _, content, _ in http.read_http_body_chunked( fconn, headers, None, "GET", 200, False)) assert chunks == ["this", "isatest", ""] connection.close()
def send_response_to_client(self, flow): if not flow.response.stream: # no streaming: # we already received the full response from the server and can send it to the client straight away. self.c.client_conn.send(flow.response.assemble()) else: # streaming: # First send the headers and then transfer the response incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False, 4096): for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() flow.response.timestamp_end = utils.timestamp()
def send_response_to_client(self, flow): if not flow.response.stream: # no streaming: # we already received the full response from the server and can send it to the client straight away. self.c.client_conn.send(flow.response.assemble()) else: # streaming: # First send the body and then transfer the response incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked( self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False, 4096): for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() flow.response.timestamp_end = utils.timestamp()
def test_stream_chunked(self): connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) connection.connect(("127.0.0.1", self.proxy.port)) fconn = connection.makefile() spec = '200:h"Transfer-Encoding"="chunked":r:b"4\\r\\nthis\\r\\n7\\r\\nisatest\\r\\n0\\r\\n\\r\\n"' connection.send("GET %s/p/%s HTTP/1.1\r\n"%(self.server.urlbase, spec)) connection.send("\r\n") httpversion, code, msg, headers, content = http.read_response(fconn, "GET", None, include_body=False) assert headers["Transfer-Encoding"][0] == 'chunked' assert code == 200 chunks = list(content for _, content, _ in http.read_http_body_chunked(fconn, headers, None, "GET", 200, False)) assert chunks == ["this", "isatest", ""] connection.close()
def handle_flow(self): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) try: req = HTTPRequest.from_stream(self.c.client_conn.rfile, body_size_limit=self.c.config.body_size_limit) self.c.log("request", "debug", [req._assemble_first_line(req.form_in)]) send_request_upstream = self.process_request(flow, req) if not send_request_upstream: return True # Be careful NOT to assign the request to the flow before # process_request completes. This is because the call can raise an # exception. If the request object is already attached, this results # in an Error object that has an attached request that has not been # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow.request) self.determine_server_address(flow, flow.request) flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow if request_reply is None or request_reply == KILL: return False if isinstance(request_reply, HTTPResponse): flow.response = request_reply else: # read initially in "stream" mode, so we can get the headers separately flow.response = self.get_response_from_server(flow.request, include_body=False) # call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True self.c.channel.ask("responseheaders", flow.response) # now get the rest of the request body, if body still needs to be read but not streaming this response if flow.response.stream: flow.response.content = CONTENT_MISSING else: flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False) # no further manipulation of self.c.server_conn beyond this point # we can safely set it as the final attribute value here. flow.server_conn = self.c.server_conn self.c.log("response", "debug", [flow.response._assemble_first_line()]) response_reply = self.c.channel.ask("response", flow.response) if response_reply is None or response_reply == KILL: return False if not flow.response.stream: # no streaming: # we already received the full response from the server and can send it to the client straight away. self.c.client_conn.send(flow.response._assemble()) else: # streaming: # First send the body and then transfer the response incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False, 4096): for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() flow.response.timestamp_end = utils.timestamp() flow.timestamp_end = utils.timestamp() close_connection = ( http.connection_close(flow.request.httpversion, flow.request.headers) or http.connection_close(flow.response.httpversion, flow.response.headers) or http.expected_http_body_size(flow.response.headers, False, flow.request.method, flow.response.code) == -1) if close_connection: if flow.request.form_in == "authority" and flow.response.code == 200: # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313: # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header pass else: return False if flow.request.form_in == "authority" and flow.response.code == 200: # TODO: Eventually add headers (space/usefulness tradeoff) # Make sure to add state info before the actual upgrade happens. # During the upgrade, we may receive an SNI indication from the client, # which resets the upstream connection. If this is the case, we must # already re-issue the CONNECT request at this point. self.c.server_conn.state.append(("http", {"state": "connect", "host": flow.request.host, "port": flow.request.port})) self.ssl_upgrade() # If the user has changed the target server on this connection, # restore the original target server flow.live.restore_server() flow.live = None return True except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: self.handle_error(e, flow)
def handle_flow(self): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.change_server) try: req = HTTPRequest.from_stream( self.c.client_conn.rfile, body_size_limit=self.c.config.body_size_limit) self.c.log("request", "debug", [req._assemble_first_line(req.form_in)]) send_request_upstream = self.process_request(flow, req) if not send_request_upstream: return True # Be careful NOT to assign the request to the flow before # process_request completes. This is because the call can raise an # exception. If the request object is already attached, this results # in an Error object that has an attached request that has not been # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow.request) self.determine_server_address(flow, flow.request) flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow if request_reply is None or request_reply == KILL: return False if isinstance(request_reply, HTTPResponse): flow.response = request_reply else: # read initially in "stream" mode, so we can get the headers separately flow.response = self.get_response_from_server( flow.request, include_body=False) # call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True self.c.channel.ask("responseheaders", flow.response) # now get the rest of the request body, if body still needs to be read but not streaming this response if flow.response.stream: flow.response.content = CONTENT_MISSING else: flow.response.content = http.read_http_body( self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False) # no further manipulation of self.c.server_conn beyond this point # we can safely set it as the final attribute value here. flow.server_conn = self.c.server_conn self.c.log("response", "debug", [flow.response._assemble_first_line()]) response_reply = self.c.channel.ask("response", flow.response) if response_reply is None or response_reply == KILL: return False if not flow.response.stream: # no streaming: # we already received the full response from the server and can send it to the client straight away. self.c.client_conn.send(flow.response._assemble()) else: # streaming: # First send the body and then transfer the response incrementally: h = flow.response._assemble_head( preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked( self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False, 4096): for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() flow.timestamp_end = utils.timestamp() close_connection = ( http.connection_close(flow.request.httpversion, flow.request.headers) or http.connection_close(flow.response.httpversion, flow.response.headers) or http.expected_http_body_size(flow.response.headers, False, flow.request.method, flow.response.code) == -1) if close_connection: if flow.request.form_in == "authority" and flow.response.code == 200: # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313: # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header pass else: return False if flow.request.form_in == "authority" and flow.response.code == 200: # TODO: Eventually add headers (space/usefulness tradeoff) # Make sure to add state info before the actual upgrade happens. # During the upgrade, we may receive an SNI indication from the client, # which resets the upstream connection. If this is the case, we must # already re-issue the CONNECT request at this point. self.c.server_conn.state.append(("http", { "state": "connect", "host": flow.request.host, "port": flow.request.port })) self.ssl_upgrade() # If the user has changed the target server on this connection, # restore the original target server self.restore_server() return True except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: self.handle_error(e, flow)