def create_request(self, method, scheme, host, port, path): """ this method creates a new artificial and minimalist request also adds it to flowlist """ c = models.ClientConnection.make_dummy(("", 0)) s = models.ServerConnection.make_dummy((host, port)) f = models.HTTPFlow(c, s) headers = models.Headers() req = models.HTTPRequest("absolute", method, scheme, host, port, path, b"HTTP/1.1", headers, b"") f.request = req self.load_flow(f) return f
def flow(self): if any(_ is None for _ in (self._request_timestamp_start, self._request_timestamp_end, self._response_timestamp_start, self._response_timestamp_end, self._timestamp_ssl_setup)): s = " ".join("%s=%s" % item for item in dict( request_start=self._request_timestamp_start, request_end=self._request_timestamp_end, response_start=self._response_timestamp_start, response_end=self._response_timestamp_end, ssl_setup=self._timestamp_ssl_setup).iteritems()) log.error(s) client_conn = models.ClientConnection.from_state( dict(address=dict(address=self._src_address, use_ipv6=False), clientcert=None, ssl_established=True, timestamp_start=self._request_timestamp_start, timestamp_end=self._request_timestamp_end, timestamp_ssl_setup=self._timestamp_ssl_setup)) server_conn = models.ServerConnection.from_state( dict( ip_address=dict(address=self._src_address, use_ipv6=False), address=dict(address=self._dst_address, use_ipv6=False), cert=None, sni=None, source_address=dict(address=self._src_address, use_ipv6=False), #peer_address=dict(address=self._src_address, # use_ipv6=False), ssl_established=True, timestamp_start=self._response_timestamp_start, timestamp_tcp_setup=self._timestamp_ssl_setup, timestamp_ssl_setup=self._timestamp_ssl_setup, timestamp_end=self._response_timestamp_end, via=None)) flow = models.HTTPFlow(client_conn, server_conn) flow.request = self.request flow.response = self.response return flow
def __call__(self): if self.mode == "transparent": self.__initial_server_tls = self.server_tls self.__initial_server_conn = self.server_conn while True: try: request = self.get_request_from_client() self.log("request", "debug", [repr(request)]) # Handle Proxy Authentication # Proxy Authentication conceptually does not work in transparent mode. # We catch this misconfiguration on startup. Here, we sort out requests # after a successful CONNECT request (which do not need to be validated anymore) if self.mode != "transparent" and not self.authenticate( request): return # Make sure that the incoming request matches our expectations self.validate_request(request) # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.first_line_format == "authority": self.handle_regular_mode_connect(request) return except netlib.exceptions.HttpReadDisconnect: # don't throw an error for disconnects that happen before/between requests. return except netlib.exceptions.NetlibException as e: self.send_error_response(400, repr(e)) six.reraise( exceptions.ProtocolException, exceptions.ProtocolException( "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) try: flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self) flow.request = request # set upstream auth if self.mode == "upstream" and self.config.upstream_auth is not None: flow.request.headers[ "Proxy-Authorization"] = self.config.upstream_auth self.process_request_hook(flow) if not flow.response: self.establish_server_connection(flow) self.get_response_from_server(flow) else: # response was set by an inline script. # we now need to emulate the responseheaders hook. flow = self.channel.ask("responseheaders", flow) self.log("response", "debug", [repr(flow.response)]) flow = self.channel.ask("response", flow) self.send_response_to_client(flow) if self.check_close_connection(flow): return # Handle 101 Switching Protocols # It may be useful to pass additional args (such as the upgrade header) # to next_layer in the future if flow.response.status_code == 101: layer = self.ctx.next_layer(self) layer() return # Upstream Proxy Mode: Handle CONNECT if flow.request.first_line_format == "authority" and flow.response.status_code == 200: self.handle_upstream_mode_connect(flow.request.copy()) return except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e: self.send_error_response(502, repr(e)) if not flow.response: flow.error = models.Error(str(e)) self.channel.ask("error", flow) self.log(traceback.format_exc(), "debug") return else: six.reraise( exceptions.ProtocolException, exceptions.ProtocolException( "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: if flow: flow.live = False
def pcap2mitm(pcapfile, mitmfile, tlsmaster=None, stream=False): try: from mitmproxy import models from mitmproxy.flow import FlowWriter from netlib.exceptions import HttpException from netlib.http import http1 except ImportError: log.warning( "In order to use this utility it is required to have the " "mitmproxy tool installed (`pip install httpreplay[mitmproxy]`)") return False if tlsmaster: tlsmaster = read_tlsmaster(tlsmaster) else: tlsmaster = {} handlers = { 443: lambda: https_handler(tlsmaster), 4443: lambda: https_handler(tlsmaster), "generic": http_handler, } reader = PcapReader(pcapfile) reader.tcp = TCPPacketStreamer(reader, handlers) writer = FlowWriter(mitmfile) l = reader.process() if not stream: # Sort the http/https requests and responses by their timestamp. l = sorted(l, key=lambda x: x[1]) for s, ts, protocol, sent, recv in l: if protocol not in ("http", "https"): continue srcip, srcport, dstip, dstport = s client_conn = models.ClientConnection.make_dummy((srcip, srcport)) client_conn.timestamp_start = ts server_conn = models.ServerConnection.make_dummy((dstip, dstport)) server_conn.timestamp_start = ts flow = models.HTTPFlow(client_conn, server_conn) try: sent = io.BytesIO(sent.raw) request = http1.read_request_head(sent) body_size = http1.expected_http_body_size(request) request.data.content = "".join( http1.read_body(sent, body_size, None)) except HttpException as e: log.warning("Error parsing HTTP request: %s", e) continue flow.request = models.HTTPRequest.wrap(request) flow.request.timestamp_start = client_conn.timestamp_start flow.request.host = dstip flow.request.port = dstport flow.request.scheme = protocol try: recv = io.BytesIO(recv.raw) response = http1.read_response_head(recv) body_size = http1.expected_http_body_size(request, response) response.data.content = "".join( http1.read_body(recv, body_size, None)) except HttpException as e: log.warning("Error parsing HTTP response: %s", e) # Fall through (?) flow.response = models.HTTPResponse.wrap(response) flow.response.timestamp_start = server_conn.timestamp_start flow.id = str( uuid.UUID(bytes=hashlib.md5( b"%d%d%s%s" % (client_conn.timestamp_start, server_conn.timestamp_start, request.data.content, response.data.content)).digest())) writer.add(flow) return True
def __call__(self): if self.mode == "transparent": self.__initial_server_tls = self.server_tls self.__initial_server_conn = self.server_conn while True: try: request = self.get_request_from_client() # Make sure that the incoming request matches our expectations self.validate_request(request) except netlib.exceptions.HttpReadDisconnect: # don't throw an error for disconnects that happen before/between requests. return except netlib.exceptions.HttpException as e: # We optimistically guess there might be an HTTP client on the # other end self.send_error_response(400, repr(e)) six.reraise( exceptions.ProtocolException, exceptions.ProtocolException( "HTTP protocol error in client request: {}".format(e) ), sys.exc_info()[2] ) self.log("request", "debug", [repr(request)]) # Handle Proxy Authentication # Proxy Authentication conceptually does not work in transparent mode. # We catch this misconfiguration on startup. Here, we sort out requests # after a successful CONNECT request (which do not need to be validated anymore) if not (self.http_authenticated or self.authenticate(request)): return flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self) flow.request = request try: # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.first_line_format == "authority": self.handle_regular_mode_connect(request) return except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e: # HTTPS tasting means that ordinary errors like resolution and # connection errors can happen here. self.send_error_response(502, repr(e)) flow.error = models.Error(str(e)) self.channel.ask("error", flow) return # update host header in reverse proxy mode if self.config.options.mode == "reverse": if six.PY2: flow.request.headers["Host"] = self.config.upstream_server.address.host.encode() else: flow.request.headers["Host"] = self.config.upstream_server.address.host # set upstream auth if self.mode == "upstream" and self.config.upstream_auth is not None: flow.request.headers["Proxy-Authorization"] = self.config.upstream_auth self.process_request_hook(flow) try: if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers): # We only support RFC6455 with WebSockets version 13 # allow inline scripts to manipulate the client handshake self.channel.ask("websockets_handshake", flow) if not flow.response: self.establish_server_connection( flow.request.host, flow.request.port, flow.request.scheme ) self.get_response_from_server(flow) else: # response was set by an inline script. # we now need to emulate the responseheaders hook. flow = self.channel.ask("responseheaders", flow) self.log("response", "debug", [repr(flow.response)]) flow = self.channel.ask("response", flow) self.send_response_to_client(flow) if self.check_close_connection(flow): return # Handle 101 Switching Protocols if flow.response.status_code == 101: return self.handle_101_switching_protocols(flow) # Upstream Proxy Mode: Handle CONNECT if flow.request.first_line_format == "authority" and flow.response.status_code == 200: self.handle_upstream_mode_connect(flow.request.copy()) return except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e: self.send_error_response(502, repr(e)) if not flow.response: flow.error = models.Error(str(e)) self.channel.ask("error", flow) return else: six.reraise(exceptions.ProtocolException, exceptions.ProtocolException( "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: if flow: flow.live = False
def pcap2mitm(pcapfile, mitmfile, tlsmaster, stream): try: from mitmproxy import models from mitmproxy.flow import FlowWriter from netlib.http import http1 from netlib.exceptions import HttpException except ImportError: raise click.Abort( "In order to use this utility it is required to have the " "mitmproxy tool installed (`pip install httpreplay[mitmproxy]`)") class NetlibHttpProtocol(Protocol): """ Like HttpProtocol, but actually covering edge-cases. """ @staticmethod def read_body(io, expected_size): """ Read a (malformed) HTTP body. Returns: A (body: bytes, is_malformed: bool) tuple. """ body_start = io.tell() try: content = b"".join(http1.read_body(io, expected_size, None)) if io.read(): # leftover? raise HttpException() return content, False except HttpException: io.seek(body_start) return io.read(), True def parse_request(self, ts, sent): try: sent = BytesIO(sent) request = http1.read_request_head(sent) body_size = http1.expected_http_body_size(request) request.data.content, malformed = self.read_body( sent, body_size) if malformed: request.headers["X-Mitmproxy-Malformed-Body"] = "1" return request except HttpException as e: log.warning("{!r} (timestamp: {})".format(e, ts)) def parse_response(self, ts, recv, request): try: recv = BytesIO(recv) response = http1.read_response_head(recv) body_size = http1.expected_http_body_size(request, response) response.data.content, malformed = self.read_body( recv, body_size) if malformed: response.headers["X-Mitmproxy-Malformed-Body"] = "1" return response except HttpException as e: log.warning("{!r} (timestamp: {})".format(e, ts)) def handle(self, s, ts, protocol, sent, recv): if protocol not in ("tcp", "tls"): self.parent.handle(s, ts, protocol, sent, recv) return req = None if sent: req = self.parse_request(ts, sent) protocols = { "tcp": "http", "tls": "https", } # Only try to decode the HTTP response if the request was valid HTTP. if req: res = self.parse_response(ts, recv, req) # Report this stream as being a valid HTTP stream. self.parent.handle(s, ts, protocols[protocol], req or sent, res) else: # This wasn't a valid HTTP stream so we forward the original TCP # or TLS stream straight ahead to our parent. self.parent.handle(s, ts, protocol, sent, recv) if tlsmaster: tlsmaster = read_tlsmaster(tlsmaster) else: tlsmaster = {} netlib_http_handler = lambda: NetlibHttpProtocol() netlib_https_handler = lambda: TLSStream(NetlibHttpProtocol(), tlsmaster) handlers = { 443: netlib_https_handler, 4443: netlib_https_handler, "generic": netlib_http_handler, } reader = PcapReader(pcapfile) reader.tcp = TCPPacketStreamer(reader, handlers) writer = FlowWriter(mitmfile) l = reader.process() if not stream: # Sort the http/https requests and responses by their timestamp. l = sorted(l, key=lambda x: x[1]) for addrs, timestamp, protocol, sent, recv in l: if protocol not in ("http", "https"): continue srcip, srcport, dstip, dstport = addrs client_conn = models.ClientConnection.make_dummy((srcip, srcport)) client_conn.timestamp_start = timestamp server_conn = models.ServerConnection.make_dummy((dstip, dstport)) server_conn.timestamp_start = timestamp flow = models.HTTPFlow(client_conn, server_conn) flow.request = models.HTTPRequest.wrap(sent) flow.request.host, flow.request.port = dstip, dstport flow.request.scheme = protocol if recv: flow.response = models.HTTPResponse.wrap(recv) writer.add(flow)