def from_stream(cls, rfile, request_method, include_content=True, body_size_limit=None): """ Parse an HTTP response from a file stream """ if not include_content: raise NotImplementedError # pragma: nocover if hasattr(rfile, "reset_timestamps"): rfile.reset_timestamps() httpversion, code, msg, headers, content = http.read_response( rfile, request_method, body_size_limit) if hasattr(rfile, "first_byte_timestamp"): timestamp_start = rfile.first_byte_timestamp else: timestamp_start = utils.timestamp() timestamp_end = utils.timestamp() return HTTPResponse(httpversion, code, msg, headers, content, timestamp_start, timestamp_end)
def from_stream(cls, rfile, request_method, include_body=True, body_size_limit=None): """ Parse an HTTP response from a file stream """ timestamp_start = utils.timestamp() if hasattr(rfile, "reset_timestamps"): rfile.reset_timestamps() httpversion, code, msg, headers, content = http.read_response( rfile, request_method, body_size_limit, include_body=include_body) if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start timestamp_start = rfile.first_byte_timestamp if include_body: timestamp_end = utils.timestamp() else: timestamp_end = None return HTTPResponse( httpversion, code, msg, headers, content, timestamp_start, timestamp_end )
def from_stream(cls, rfile, request_method, include_body=True, body_size_limit=None): """ Parse an HTTP response from a file stream """ timestamp_start = utils.timestamp() if hasattr(rfile, "reset_timestamps"): rfile.reset_timestamps() httpversion, code, msg, headers, content = http.read_response( rfile, request_method, body_size_limit, include_body=include_body) if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start timestamp_start = rfile.first_byte_timestamp if include_body: timestamp_end = utils.timestamp() else: timestamp_end = None return HTTPResponse(httpversion, code, msg, headers, content, timestamp_start, timestamp_end)
def from_stream(cls, rfile, include_content=True, body_size_limit=None): """ Parse an HTTP request from a file stream """ httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \ = None, None, None, None, None, None, None, None, None, None if hasattr(rfile, "reset_timestamps"): rfile.reset_timestamps() request_line = get_line(rfile) if hasattr(rfile, "first_byte_timestamp"): timestamp_start = rfile.first_byte_timestamp else: timestamp_start = utils.timestamp() request_line_parts = http.parse_init(request_line) if not request_line_parts: raise http.HttpError( 400, "Bad HTTP request line: %s" % repr(request_line)) method, path, httpversion = request_line_parts if path == '*': form_in = "asterisk" elif path.startswith("/"): form_in = "origin" if not netlib.utils.isascii(path): raise http.HttpError( 400, "Bad HTTP request line: %s" % repr(request_line)) elif method.upper() == 'CONNECT': form_in = "authority" r = http.parse_init_connect(request_line) if not r: raise http.HttpError( 400, "Bad HTTP request line: %s" % repr(request_line)) host, port, _ = r path = None else: form_in = "absolute" r = http.parse_init_proxy(request_line) if not r: raise http.HttpError( 400, "Bad HTTP request line: %s" % repr(request_line)) _, scheme, host, port, path, _ = r headers = http.read_headers(rfile) if headers is None: raise http.HttpError(400, "Invalid headers") if include_content: content = http.read_http_body(rfile, headers, body_size_limit, True) timestamp_end = utils.timestamp() return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers, content, timestamp_start, timestamp_end)
def __init__(self, httpversion, headers, content, timestamp_start=None, timestamp_end=None): self.httpversion = httpversion self.headers = headers """@type: ODictCaseless""" self.content = content self.timestamp_start = timestamp_start if timestamp_start is not None else utils.timestamp() self.timestamp_end = timestamp_end if timestamp_end is not None else utils.timestamp()
def __init__(self, httpversion, headers, content, timestamp_start=None, timestamp_end=None): self.httpversion = httpversion self.headers = headers """@type: ODictCaseless""" self.content = content self.timestamp_start = timestamp_start if timestamp_start is not None else utils.timestamp() self.timestamp_end = timestamp_end if timestamp_end is not None else utils.timestamp() self.flow = None # will usually be set by the flow backref mixin """@type: HTTPFlow"""
def from_stream(cls, rfile, include_content=True, body_size_limit=None): """ Parse an HTTP request from a file stream """ httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \ = None, None, None, None, None, None, None, None, None, None if hasattr(rfile, "reset_timestamps"): rfile.reset_timestamps() request_line = get_line(rfile) if hasattr(rfile, "first_byte_timestamp"): timestamp_start = rfile.first_byte_timestamp else: timestamp_start = utils.timestamp() request_line_parts = http.parse_init(request_line) if not request_line_parts: raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) method, path, httpversion = request_line_parts if path == '*': form_in = "asterisk" elif path.startswith("/"): form_in = "origin" if not netlib.utils.isascii(path): raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) elif method.upper() == 'CONNECT': form_in = "authority" r = http.parse_init_connect(request_line) if not r: raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) host, port, _ = r path = None else: form_in = "absolute" r = http.parse_init_proxy(request_line) if not r: raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) _, scheme, host, port, path, _ = r headers = http.read_headers(rfile) if headers is None: raise http.HttpError(400, "Invalid headers") if include_content: content = http.read_http_body(rfile, headers, body_size_limit, True) timestamp_end = utils.timestamp() return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers, content, timestamp_start, timestamp_end)
def __init__(self, httpversion, headers, content, timestamp_start=None, timestamp_end=None): self.httpversion = httpversion self.headers = headers """@type: ODictCaseless""" self.content = content self.timestamp_start = timestamp_start if timestamp_start is not None else utils.timestamp( ) self.timestamp_end = timestamp_end if timestamp_end is not None else utils.timestamp( )
class HTTPHandler(ProtocolHandler): """ HTTPHandler implements mitmproxys understanding of the HTTP protocol. """ def __init__(self, c): super(HTTPHandler, self).__init__(c) self.expected_form_in = c.config.mode.http_form_in self.expected_form_out = c.config.mode.http_form_out self.skip_authentication = False def handle_messages(self): while self.handle_flow(): pass def get_response_from_server(self, flow): self.c.establish_server_connection() request_raw = flow.request.assemble() for attempt in (0, 1): try: self.c.server_conn.send(request_raw) # Only get the headers at first... flow.response = HTTPResponse.from_stream( self.c.server_conn.rfile, flow.request.method, body_size_limit=self.c.config.body_size_limit, include_body=False) break except (tcp.NetLibDisconnect, http.HttpErrorConnClosed), v: self.c.log("error in server communication: %s" % repr(v), level="debug") if attempt == 0: # In any case, we try to reconnect at least once. # This is necessary because it might be possible that we already initiated an upstream connection # after clientconnect that has already been expired, e.g consider the following event log: # > clientconnect (transparent mode destination known) # > serverconnect # > read n% of large request # > server detects timeout, disconnects # > read (100-n)% of large request # > send large request upstream self.c.server_reconnect() else: raise # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True self.c.channel.ask("responseheaders", flow) # now get the rest of the request body, if body still needs to be read # but not streaming this response if flow.response.stream: flow.response.content = CONTENT_MISSING else: flow.response.content = http.read_http_body( self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False) flow.response.timestamp_end = utils.timestamp()
def handle_flow(self): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.change_server) try: req = HTTPRequest.from_stream( self.c.client_conn.rfile, body_size_limit=self.c.config.body_size_limit) self.c.log("request", "debug", [req._assemble_first_line(req.form_in)]) send_upstream = self.process_request(flow, req) if not send_upstream: return True # Be careful NOT to assign the request to the flow before # process_request completes. This is because the call can raise an # exception. If the request object is already attached, this results # in an Error object that has an attached request that has not been # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow.request) flow.server_conn = self.c.server_conn if request_reply is None or request_reply == KILL: return False if isinstance(request_reply, HTTPResponse): flow.response = request_reply else: flow.response = self.get_response_from_server(flow.request) flow.server_conn = self.c.server_conn # no further manipulation of self.c.server_conn beyond this point # we can safely set it as the final attribute value here. self.c.log("response", "debug", [flow.response._assemble_first_line()]) response_reply = self.c.channel.ask("response", flow.response) if response_reply is None or response_reply == KILL: return False self.c.client_conn.send(flow.response._assemble()) flow.timestamp_end = utils.timestamp() if (http.connection_close(flow.request.httpversion, flow.request.headers) or http.connection_close(flow.response.httpversion, flow.response.headers)): return False if flow.request.form_in == "authority": self.ssl_upgrade() # If the user has changed the target server on this connection, # restore the original target server self.restore_server() return True except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: self.handle_error(e, flow)
def handle_flow(self): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.change_server) try: req = HTTPRequest.from_stream(self.c.client_conn.rfile, body_size_limit=self.c.config.body_size_limit) self.c.log("request", "debug", [req._assemble_first_line(req.form_in)]) send_request_upstream = self.process_request(flow, req) if not send_request_upstream: return True # Be careful NOT to assign the request to the flow before # process_request completes. This is because the call can raise an # exception. If the request object is already attached, this results # in an Error object that has an attached request that has not been # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow.request) flow.server_conn = self.c.server_conn if request_reply is None or request_reply == KILL: return False if isinstance(request_reply, HTTPResponse): flow.response = request_reply else: flow.response = self.get_response_from_server(flow.request) flow.server_conn = self.c.server_conn # no further manipulation of self.c.server_conn beyond this point # we can safely set it as the final attribute value here. self.c.log("response", "debug", [flow.response._assemble_first_line()]) response_reply = self.c.channel.ask("response", flow.response) if response_reply is None or response_reply == KILL: return False self.c.client_conn.send(flow.response._assemble()) flow.timestamp_end = utils.timestamp() if (http.connection_close(flow.request.httpversion, flow.request.headers) or http.connection_close(flow.response.httpversion, flow.response.headers)): if flow.request.form_in == "authority" and flow.response.code == 200: # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313: # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header pass else: return False if flow.request.form_in == "authority" and flow.response.code == 200: self.ssl_upgrade() # If the user has changed the target server on this connection, # restore the original target server self.restore_server() return True except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: self.handle_error(e, flow)
def send_response_to_client(self, flow): if not flow.response.stream: # no streaming: # we already received the full response from the server and can send it to the client straight away. self.c.client_conn.send(flow.response.assemble()) else: # streaming: # First send the headers and then transfer the response incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False, 4096): for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() flow.response.timestamp_end = utils.timestamp()
def send_response_to_client(self, flow): if not flow.response.stream: # no streaming: # we already received the full response from the server and can send it to the client straight away. self.c.client_conn.send(flow.response.assemble()) else: # streaming: # First send the body and then transfer the response incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked( self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False, 4096): for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() flow.response.timestamp_end = utils.timestamp()
def send_response_to_client(self, flow): if not flow.response.stream: # no streaming: # we already received the full response from the server and can # send it to the client straight away. self.c.client_conn.send(self.c.client_conn.protocol.assemble(flow.response)) else: if isinstance(self.c.client_conn.protocol, http2.HTTP2Protocol): raise NotImplementedError("HTTP streaming with HTTP/2 is currently not supported.") # streaming: # First send the headers and then transfer the response # incrementally: h = self.c.client_conn.protocol._assemble_response_first_line(flow.response) self.c.client_conn.send(h + "\r\n") h = self.c.client_conn.protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True) self.c.client_conn.send(h + "\r\n") chunks = self.c.server_conn.protocol.read_http_body_chunked( flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False, 4096 ) if callable(flow.response.stream): chunks = flow.response.stream(chunks) for chunk in chunks: for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() flow.response.timestamp_end = utils.timestamp()
def handle_flow(self): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) try: req = HTTPRequest.from_stream(self.c.client_conn.rfile, body_size_limit=self.c.config.body_size_limit) self.c.log("request", "debug", [req._assemble_first_line(req.form_in)]) send_request_upstream = self.process_request(flow, req) if not send_request_upstream: return True # Be careful NOT to assign the request to the flow before # process_request completes. This is because the call can raise an # exception. If the request object is already attached, this results # in an Error object that has an attached request that has not been # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow.request) self.determine_server_address(flow, flow.request) flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow if request_reply is None or request_reply == KILL: return False if isinstance(request_reply, HTTPResponse): flow.response = request_reply else: # read initially in "stream" mode, so we can get the headers separately flow.response = self.get_response_from_server(flow.request, include_body=False) # call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True self.c.channel.ask("responseheaders", flow.response) # now get the rest of the request body, if body still needs to be read but not streaming this response if flow.response.stream: flow.response.content = CONTENT_MISSING else: flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False) # no further manipulation of self.c.server_conn beyond this point # we can safely set it as the final attribute value here. flow.server_conn = self.c.server_conn self.c.log("response", "debug", [flow.response._assemble_first_line()]) response_reply = self.c.channel.ask("response", flow.response) if response_reply is None or response_reply == KILL: return False if not flow.response.stream: # no streaming: # we already received the full response from the server and can send it to the client straight away. self.c.client_conn.send(flow.response._assemble()) else: # streaming: # First send the body and then transfer the response incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False, 4096): for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() flow.response.timestamp_end = utils.timestamp() flow.timestamp_end = utils.timestamp() close_connection = ( http.connection_close(flow.request.httpversion, flow.request.headers) or http.connection_close(flow.response.httpversion, flow.response.headers) or http.expected_http_body_size(flow.response.headers, False, flow.request.method, flow.response.code) == -1) if close_connection: if flow.request.form_in == "authority" and flow.response.code == 200: # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313: # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header pass else: return False if flow.request.form_in == "authority" and flow.response.code == 200: # TODO: Eventually add headers (space/usefulness tradeoff) # Make sure to add state info before the actual upgrade happens. # During the upgrade, we may receive an SNI indication from the client, # which resets the upstream connection. If this is the case, we must # already re-issue the CONNECT request at this point. self.c.server_conn.state.append(("http", {"state": "connect", "host": flow.request.host, "port": flow.request.port})) self.ssl_upgrade() # If the user has changed the target server on this connection, # restore the original target server flow.live.restore_server() flow.live = None return True except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: self.handle_error(e, flow)
# already re-issue the CONNECT request at this point. self.c.server_conn.state.append(("http", {"state": "connect", "host": flow.request.host, "port": flow.request.port})) if not self.process_connect_request((flow.request.host, flow.request.port)): return False # If the user has changed the target server on this connection, # restore the original target server flow.live.restore_server() return True # Next flow please. except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: self.handle_error(e, flow) finally: flow.timestamp_end = utils.timestamp() flow.live = None # Connection is not live anymore. return False def handle_server_reconnect(self, state): if state["state"] == "connect": send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False) else: # pragma: nocover raise RuntimeError("Unknown State: %s" % state["state"]) def handle_error(self, error, flow=None): message = repr(error) message_debug = None if isinstance(error, tcp.NetLibDisconnect):
def handle_flow(self): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.change_server) try: req = HTTPRequest.from_stream( self.c.client_conn.rfile, body_size_limit=self.c.config.body_size_limit) self.c.log("request", "debug", [req._assemble_first_line(req.form_in)]) send_request_upstream = self.process_request(flow, req) if not send_request_upstream: return True # Be careful NOT to assign the request to the flow before # process_request completes. This is because the call can raise an # exception. If the request object is already attached, this results # in an Error object that has an attached request that has not been # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow.request) self.determine_server_address(flow, flow.request) flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow if request_reply is None or request_reply == KILL: return False if isinstance(request_reply, HTTPResponse): flow.response = request_reply else: # read initially in "stream" mode, so we can get the headers separately flow.response = self.get_response_from_server( flow.request, include_body=False) # call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True self.c.channel.ask("responseheaders", flow.response) # now get the rest of the request body, if body still needs to be read but not streaming this response if flow.response.stream: flow.response.content = CONTENT_MISSING else: flow.response.content = http.read_http_body( self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False) # no further manipulation of self.c.server_conn beyond this point # we can safely set it as the final attribute value here. flow.server_conn = self.c.server_conn self.c.log("response", "debug", [flow.response._assemble_first_line()]) response_reply = self.c.channel.ask("response", flow.response) if response_reply is None or response_reply == KILL: return False if not flow.response.stream: # no streaming: # we already received the full response from the server and can send it to the client straight away. self.c.client_conn.send(flow.response._assemble()) else: # streaming: # First send the body and then transfer the response incrementally: h = flow.response._assemble_head( preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked( self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False, 4096): for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() flow.timestamp_end = utils.timestamp() close_connection = ( http.connection_close(flow.request.httpversion, flow.request.headers) or http.connection_close(flow.response.httpversion, flow.response.headers) or http.expected_http_body_size(flow.response.headers, False, flow.request.method, flow.response.code) == -1) if close_connection: if flow.request.form_in == "authority" and flow.response.code == 200: # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313: # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header pass else: return False if flow.request.form_in == "authority" and flow.response.code == 200: # TODO: Eventually add headers (space/usefulness tradeoff) # Make sure to add state info before the actual upgrade happens. # During the upgrade, we may receive an SNI indication from the client, # which resets the upstream connection. If this is the case, we must # already re-issue the CONNECT request at this point. self.c.server_conn.state.append(("http", { "state": "connect", "host": flow.request.host, "port": flow.request.port })) self.ssl_upgrade() # If the user has changed the target server on this connection, # restore the original target server self.restore_server() return True except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: self.handle_error(e, flow)
"port": flow.request.port })) if not self.process_connect_request( (flow.request.host, flow.request.port)): return False # If the user has changed the target server on this connection, # restore the original target server flow.live.restore_server() return True # Next flow please. except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: self.handle_error(e, flow) finally: flow.timestamp_end = utils.timestamp() flow.live = None # Connection is not live anymore. return False def handle_server_reconnect(self, state): if state["state"] == "connect": send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False) else: # pragma: nocover raise RuntimeError("Unknown State: %s" % state["state"]) def handle_error(self, error, flow=None): message = repr(error)
def get_response_from_server(self, flow): self.c.establish_server_connection() for attempt in (0, 1): try: if not self.c.server_conn.protocol: # instantiate new protocol if connection does not have one yet # TODO: select correct protocol based on ALPN (?) self.c.server_conn.protocol = http1.HTTP1Protocol(self.c.server_conn) # self.c.server_conn.protocol = http2.HTTP2Protocol(self.c.server_conn) # self.c.server_conn.protocol.perform_connection_preface() self.c.server_conn.send(self.c.server_conn.protocol.assemble(flow.request)) # Only get the headers at first... flow.response = HTTPResponse.from_protocol( self.c.server_conn.protocol, flow.request.method, body_size_limit=self.c.config.body_size_limit, include_body=False, ) break except (tcp.NetLibError, http.HttpErrorConnClosed) as v: self.c.log( "error in server communication: %s" % repr(v), level="debug" ) if attempt == 0: # In any case, we try to reconnect at least once. This is # necessary because it might be possible that we already # initiated an upstream connection after clientconnect that # has already been expired, e.g consider the following event # log: # > clientconnect (transparent mode destination known) # > serverconnect # > read n% of large request # > server detects timeout, disconnects # > read (100-n)% of large request # > send large request upstream self.c.server_reconnect() else: raise # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.c.channel.ask("responseheaders", flow) if flow is None or flow == KILL: raise KillSignal() else: # now get the rest of the request body, if body still needs to be # read but not streaming this response if flow.response.stream: flow.response.content = CONTENT_MISSING else: if isinstance(self.c.server_conn.protocol, http1.HTTP1Protocol): # streaming is only supported with HTTP/1 at the moment flow.response.content = self.c.server_conn.protocol.read_http_body( flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False ) flow.response.timestamp_end = utils.timestamp()