Пример #1
0
    def handle_101_switching_protocols(self, flow):
        """
        Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request.
        """
        # Check for WebSockets handshake
        is_websockets = (flow
                         and websockets.check_handshake(flow.request.headers)
                         and websockets.check_handshake(flow.response.headers))
        if is_websockets and not self.config.options.websockets:
            self.log(
                "Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.",
                "info")

        if is_websockets and self.config.options.websockets:
            layer = pwebsockets.WebSocketsLayer(self, flow)
        else:
            layer = self.ctx.next_layer(self)

        layer()
Пример #2
0
    def handle_101_switching_protocols(self, flow):
        """
        Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request.
        """
        # Check for WebSockets handshake
        is_websockets = (
            flow and
            websockets.check_handshake(flow.request.headers) and
            websockets.check_handshake(flow.response.headers)
        )
        if is_websockets and not self.config.options.websockets:
            self.log(
                "Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.",
                "info"
            )

        if is_websockets and self.config.options.websockets:
            layer = pwebsockets.WebSocketsLayer(self, flow)
        else:
            layer = self.ctx.next_layer(self)

        layer()
Пример #3
0
    def _setup_connection(self):
        client = netlib.tcp.TCPClient(("127.0.0.1", self.proxy.port))
        client.connect()

        request = http.Request(
            "authority",
            "CONNECT",
            "",
            "localhost",
            self.server.server.address.port,
            "",
            "HTTP/1.1",
            content=b'')
        client.wfile.write(http.http1.assemble_request(request))
        client.wfile.flush()

        response = http.http1.read_response(client.rfile, request)

        if self.ssl:
            client.convert_to_ssl()
            assert client.ssl_established

        request = http.Request(
            "relative",
            "GET",
            "http",
            "localhost",
            self.server.server.address.port,
            "/ws",
            "HTTP/1.1",
            headers=http.Headers(
                connection="upgrade",
                upgrade="websocket",
                sec_websocket_version="13",
                sec_websocket_key="1234",
            ),
            content=b'')
        client.wfile.write(http.http1.assemble_request(request))
        client.wfile.flush()

        response = http.http1.read_response(client.rfile, request)
        assert websockets.check_handshake(response.headers)

        return client
Пример #4
0
    def _setup_connection(self):
        client = netlib.tcp.TCPClient(("127.0.0.1", self.proxy.port))
        client.connect()

        request = http.Request("authority",
                               "CONNECT",
                               "",
                               "localhost",
                               self.server.server.address.port,
                               "",
                               "HTTP/1.1",
                               content=b'')
        client.wfile.write(http.http1.assemble_request(request))
        client.wfile.flush()

        response = http.http1.read_response(client.rfile, request)

        if self.ssl:
            client.convert_to_ssl()
            assert client.ssl_established

        request = http.Request("relative",
                               "GET",
                               "http",
                               "localhost",
                               self.server.server.address.port,
                               "/ws",
                               "HTTP/1.1",
                               headers=http.Headers(
                                   connection="upgrade",
                                   upgrade="websocket",
                                   sec_websocket_version="13",
                                   sec_websocket_key="1234",
                               ),
                               content=b'')
        client.wfile.write(http.http1.assemble_request(request))
        client.wfile.flush()

        response = http.http1.read_response(client.rfile, request)
        assert websockets.check_handshake(response.headers)

        return client
Пример #5
0
        def handle(self):
            try:
                request = http.http1.read_request(self.rfile)
                assert websockets.check_handshake(request.headers)

                response = http.Response(
                    "HTTP/1.1",
                    101,
                    reason=http.status_codes.RESPONSES.get(101),
                    headers=http.Headers(
                        connection='upgrade',
                        upgrade='websocket',
                        sec_websocket_accept=b'',
                    ),
                    content=b'',
                )
                self.wfile.write(http.http1.assemble_response(response))
                self.wfile.flush()

                self.server.handle_websockets(self.rfile, self.wfile)
            except:
                traceback.print_exc()
Пример #6
0
        def handle(self):
            try:
                request = http.http1.read_request(self.rfile)
                assert websockets.check_handshake(request.headers)

                response = http.Response(
                    "HTTP/1.1",
                    101,
                    reason=http.status_codes.RESPONSES.get(101),
                    headers=http.Headers(
                        connection='upgrade',
                        upgrade='websocket',
                        sec_websocket_accept=b'',
                    ),
                    content=b'',
                )
                self.wfile.write(http.http1.assemble_response(response))
                self.wfile.flush()

                self.server.handle_websockets(self.rfile, self.wfile)
            except:
                traceback.print_exc()
Пример #7
0
    def __call__(self):
        if self.mode == "transparent":
            self.__initial_server_tls = self.server_tls
            self.__initial_server_conn = self.server_conn
        while True:
            try:
                request = self.get_request_from_client()
                # Make sure that the incoming request matches our expectations
                self.validate_request(request)
            except netlib.exceptions.HttpReadDisconnect:
                # don't throw an error for disconnects that happen before/between requests.
                return
            except netlib.exceptions.HttpException as e:
                # We optimistically guess there might be an HTTP client on the
                # other end
                self.send_error_response(400, repr(e))
                self.log(
                    "request",
                    "warn",
                    "HTTP protocol error in client request: %s" % e
                )
                return

            self.log("request", "debug", [repr(request)])

            # Handle Proxy Authentication
            # Proxy Authentication conceptually does not work in transparent mode.
            # We catch this misconfiguration on startup. Here, we sort out requests
            # after a successful CONNECT request (which do not need to be validated anymore)
            if not (self.http_authenticated or self.authenticate(request)):
                return

            flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self)
            flow.request = request

            try:
                # Regular Proxy Mode: Handle CONNECT
                if self.mode == "regular" and request.first_line_format == "authority":
                    self.handle_regular_mode_connect(request)
                    return
            except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e:
                # HTTPS tasting means that ordinary errors like resolution and
                # connection errors can happen here.
                self.send_error_response(502, repr(e))
                flow.error = models.Error(str(e))
                self.channel.ask("error", flow)
                return

            # set upstream auth
            if self.mode == "upstream" and self.config.upstream_auth is not None:
                flow.request.headers["Proxy-Authorization"] = self.config.upstream_auth
            self.process_request_hook(flow)

            try:
                if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
                    # we only support RFC6455 with WebSockets version 13
                    # allow inline scripts to manupulate the client handshake
                    self.channel.ask("websockets_handshake", flow)

                if not flow.response:
                    self.establish_server_connection(
                        flow.request.host,
                        flow.request.port,
                        flow.request.scheme
                    )
                    self.get_response_from_server(flow)
                else:
                    # response was set by an inline script.
                    # we now need to emulate the responseheaders hook.
                    flow = self.channel.ask("responseheaders", flow)

                self.log("response", "debug", [repr(flow.response)])
                flow = self.channel.ask("response", flow)
                self.send_response_to_client(flow)

                if self.check_close_connection(flow):
                    return

                # Handle 101 Switching Protocols
                # It may be useful to pass additional args (such as the upgrade header)
                # to next_layer in the future
                if flow.response.status_code == 101:
                    layer = self.ctx.next_layer(self, flow)
                    layer()
                    return

                # Upstream Proxy Mode: Handle CONNECT
                if flow.request.first_line_format == "authority" and flow.response.status_code == 200:
                    self.handle_upstream_mode_connect(flow.request.copy())
                    return

            except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e:
                self.send_error_response(502, repr(e))
                if not flow.response:
                    flow.error = models.Error(str(e))
                    self.channel.ask("error", flow)
                    return
                else:
                    six.reraise(exceptions.ProtocolException, exceptions.ProtocolException(
                        "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
            finally:
                if flow:
                    flow.live = False
Пример #8
0
    def handle_http_request(self, logger):
        """
            Returns a (handler, log) tuple.

            handler: Handler for the next request, or None to disconnect
            log: A dictionary, or None
        """
        with logger.ctx() as lg:
            try:
                req = self.protocol.read_request(self.rfile)
            except HttpReadDisconnect:
                return None, None
            except HttpException as s:
                s = str(s)
                lg(s)
                return None, dict(type="error", msg=s)

            if req.method == 'CONNECT':
                return self.protocol.handle_http_connect(
                    [req.host, req.port, req.http_version], lg)

            method = req.method
            path = req.path
            http_version = req.http_version
            headers = req.headers

            clientcert = None
            if self.clientcert:
                clientcert = dict(
                    cn=self.clientcert.cn,
                    subject=self.clientcert.subject,
                    serial=self.clientcert.serial,
                    notbefore=self.clientcert.notbefore.isoformat(),
                    notafter=self.clientcert.notafter.isoformat(),
                    keyinfo=self.clientcert.keyinfo,
                )

            retlog = dict(
                type="crafted",
                protocol="http",
                request=dict(
                    path=path,
                    method=method,
                    headers=headers.fields,
                    http_version=http_version,
                    sni=self.sni,
                    remote_address=self.address(),
                    clientcert=clientcert,
                ),
                cipher=None,
            )
            if self.ssl_established:
                retlog["cipher"] = self.get_current_cipher()

            m = utils.MemBool()

            valid_websocket_handshake = websockets.check_handshake(headers)
            self.settings.websocket_key = websockets.get_client_key(headers)

            # If this is a websocket initiation, we respond with a proper
            # server response, unless over-ridden.
            if valid_websocket_handshake:
                anchor_gen = language.parse_pathod("ws")
            else:
                anchor_gen = None

            for regex, spec in self.server.anchors:
                if regex.match(path):
                    anchor_gen = language.parse_pathod(spec, self.use_http2)
                    break
            else:
                if m(path.startswith(self.server.craftanchor)):
                    spec = urllib.parse.unquote(
                        path)[len(self.server.craftanchor):]
                    if spec:
                        try:
                            anchor_gen = language.parse_pathod(
                                spec, self.use_http2)
                        except language.ParseException as v:
                            lg("Parse error: %s" % v.msg)
                            anchor_gen = iter([
                                self.make_http_error_response(
                                    "Parse Error",
                                    "Error parsing response spec: %s\n" %
                                    (v.msg + v.marked()))
                            ])
                else:
                    if self.use_http2:
                        anchor_gen = iter([
                            self.make_http_error_response(
                                "Spec Error",
                                "HTTP/2 only supports request/response with the craft anchor point: %s"
                                % self.server.craftanchor)
                        ])

            if not anchor_gen:
                anchor_gen = iter([
                    self.make_http_error_response(
                        "Not found", "No valid craft request found")
                ])

            spec = next(anchor_gen)

            if self.use_http2 and isinstance(spec, language.http2.Response):
                spec.stream_id = req.stream_id

            lg("crafting spec: %s" % spec)
            nexthandler, retlog["response"] = self.http_serve_crafted(spec, lg)
            if nexthandler and valid_websocket_handshake:
                self.protocol = protocols.websockets.WebsocketsProtocol(self)
                return self.protocol.handle_websocket, retlog
            else:
                return nexthandler, retlog
Пример #9
0
    def handle_http_request(self, logger):
        """
            Returns a (handler, log) tuple.

            handler: Handler for the next request, or None to disconnect
            log: A dictionary, or None
        """
        with logger.ctx() as lg:
            try:
                req = self.protocol.read_request(self.rfile)
            except HttpReadDisconnect:
                return None, None
            except HttpException as s:
                s = str(s)
                lg(s)
                return None, dict(type="error", msg=s)

            if req.method == 'CONNECT':
                return self.protocol.handle_http_connect([req.host, req.port, req.http_version], lg)

            method = req.method
            path = req.path
            http_version = req.http_version
            headers = req.headers

            clientcert = None
            if self.clientcert:
                clientcert = dict(
                    cn=self.clientcert.cn,
                    subject=self.clientcert.subject,
                    serial=self.clientcert.serial,
                    notbefore=self.clientcert.notbefore.isoformat(),
                    notafter=self.clientcert.notafter.isoformat(),
                    keyinfo=self.clientcert.keyinfo,
                )

            retlog = dict(
                type="crafted",
                protocol="http",
                request=dict(
                    path=path,
                    method=method,
                    headers=headers.fields,
                    http_version=http_version,
                    sni=self.sni,
                    remote_address=self.address(),
                    clientcert=clientcert,
                ),
                cipher=None,
            )
            if self.ssl_established:
                retlog["cipher"] = self.get_current_cipher()

            m = utils.MemBool()

            valid_websockets_handshake = websockets.check_handshake(headers)
            self.settings.websocket_key = websockets.get_client_key(headers)

            # If this is a websocket initiation, we respond with a proper
            # server response, unless over-ridden.
            if valid_websockets_handshake:
                anchor_gen = language.parse_pathod("ws")
            else:
                anchor_gen = None

            for regex, spec in self.server.anchors:
                if regex.match(path):
                    anchor_gen = language.parse_pathod(spec, self.use_http2)
                    break
            else:
                if m(path.startswith(self.server.craftanchor)):
                    spec = urllib.parse.unquote(path)[len(self.server.craftanchor):]
                    if spec:
                        try:
                            anchor_gen = language.parse_pathod(spec, self.use_http2)
                        except language.ParseException as v:
                            lg("Parse error: %s" % v.msg)
                            anchor_gen = iter([self.make_http_error_response(
                                "Parse Error",
                                "Error parsing response spec: %s\n" % (
                                    v.msg + v.marked()
                                )
                            )])
                else:
                    if self.use_http2:
                        anchor_gen = iter([self.make_http_error_response(
                            "Spec Error",
                            "HTTP/2 only supports request/response with the craft anchor point: %s" %
                            self.server.craftanchor
                        )])

            if not anchor_gen:
                anchor_gen = iter([self.make_http_error_response(
                    "Not found",
                    "No valid craft request found"
                )])

            spec = next(anchor_gen)

            if self.use_http2 and isinstance(spec, language.http2.Response):
                spec.stream_id = req.stream_id

            lg("crafting spec: %s" % spec)
            nexthandler, retlog["response"] = self.http_serve_crafted(
                spec,
                lg
            )
            if nexthandler and valid_websockets_handshake:
                self.protocol = protocols.websockets.WebsocketsProtocol(self)
                return self.protocol.handle_websocket, retlog
            else:
                return nexthandler, retlog
Пример #10
0
    def __call__(self):
        if self.mode == "transparent":
            self.__initial_server_tls = self.server_tls
            self.__initial_server_conn = self.server_conn
        while True:
            try:
                request = self.get_request_from_client()
                # Make sure that the incoming request matches our expectations
                self.validate_request(request)
            except netlib.exceptions.HttpReadDisconnect:
                # don't throw an error for disconnects that happen before/between requests.
                return
            except netlib.exceptions.HttpException as e:
                # We optimistically guess there might be an HTTP client on the
                # other end
                self.send_error_response(400, repr(e))
                six.reraise(
                    exceptions.ProtocolException,
                    exceptions.ProtocolException(
                        "HTTP protocol error in client request: {}".format(e)
                    ),
                    sys.exc_info()[2]
                )

            self.log("request", "debug", [repr(request)])

            # Handle Proxy Authentication
            # Proxy Authentication conceptually does not work in transparent mode.
            # We catch this misconfiguration on startup. Here, we sort out requests
            # after a successful CONNECT request (which do not need to be validated anymore)
            if not (self.http_authenticated or self.authenticate(request)):
                return

            flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self)
            flow.request = request

            try:
                # Regular Proxy Mode: Handle CONNECT
                if self.mode == "regular" and request.first_line_format == "authority":
                    self.handle_regular_mode_connect(request)
                    return
            except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e:
                # HTTPS tasting means that ordinary errors like resolution and
                # connection errors can happen here.
                self.send_error_response(502, repr(e))
                flow.error = models.Error(str(e))
                self.channel.ask("error", flow)
                return

            # update host header in reverse proxy mode
            if self.config.options.mode == "reverse":
                if six.PY2:
                    flow.request.headers["Host"] = self.config.upstream_server.address.host.encode()
                else:
                    flow.request.headers["Host"] = self.config.upstream_server.address.host

            # set upstream auth
            if self.mode == "upstream" and self.config.upstream_auth is not None:
                flow.request.headers["Proxy-Authorization"] = self.config.upstream_auth
            self.process_request_hook(flow)

            try:
                if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
                    # We only support RFC6455 with WebSockets version 13
                    # allow inline scripts to manipulate the client handshake
                    self.channel.ask("websockets_handshake", flow)

                if not flow.response:
                    self.establish_server_connection(
                        flow.request.host,
                        flow.request.port,
                        flow.request.scheme
                    )
                    self.get_response_from_server(flow)
                else:
                    # response was set by an inline script.
                    # we now need to emulate the responseheaders hook.
                    flow = self.channel.ask("responseheaders", flow)

                self.log("response", "debug", [repr(flow.response)])
                flow = self.channel.ask("response", flow)
                self.send_response_to_client(flow)

                if self.check_close_connection(flow):
                    return

                # Handle 101 Switching Protocols
                if flow.response.status_code == 101:
                    return self.handle_101_switching_protocols(flow)

                # Upstream Proxy Mode: Handle CONNECT
                if flow.request.first_line_format == "authority" and flow.response.status_code == 200:
                    self.handle_upstream_mode_connect(flow.request.copy())
                    return

            except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e:
                self.send_error_response(502, repr(e))
                if not flow.response:
                    flow.error = models.Error(str(e))
                    self.channel.ask("error", flow)
                    return
                else:
                    six.reraise(exceptions.ProtocolException, exceptions.ProtocolException(
                        "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
            finally:
                if flow:
                    flow.live = False
Пример #11
0
 def test_check_handshake(self, input, expected):
     h = http.Headers(input)
     assert websockets.check_handshake(h) == expected
Пример #12
0
    def _next_layer(self, top_layer, flow):
        if flow is not None:
            # We already have a flow, try to derive the next information from it

            # Check for WebSockets handshake
            is_websockets = (
                flow and
                websockets.check_handshake(flow.request.headers) and
                websockets.check_handshake(flow.response.headers)
            )
            if isinstance(top_layer, protocol.HttpLayer) and is_websockets:
                return protocol.WebSocketsLayer(top_layer, flow)

        try:
            d = top_layer.client_conn.rfile.peek(3)
        except netlib.exceptions.TcpException as e:
            six.reraise(exceptions.ProtocolException, exceptions.ProtocolException(str(e)), sys.exc_info()[2])
        client_tls = protocol.is_tls_record_magic(d)

        # 1. check for --ignore
        if self.config.check_ignore:
            ignore = self.config.check_ignore(top_layer.server_conn.address)
            if not ignore and client_tls:
                try:
                    client_hello = protocol.TlsClientHello.from_client_conn(self.client_conn)
                except exceptions.TlsProtocolException as e:
                    self.log("Cannot parse Client Hello: %s" % repr(e), "error")
                else:
                    ignore = self.config.check_ignore((client_hello.sni, 443))
            if ignore:
                return protocol.RawTCPLayer(top_layer, ignore=True)

        # 2. Always insert a TLS layer, even if there's neither client nor server tls.
        # An inline script may upgrade from http to https,
        # in which case we need some form of TLS layer.
        if isinstance(top_layer, modes.ReverseProxy):
            return protocol.TlsLayer(top_layer, client_tls, top_layer.server_tls)
        if isinstance(top_layer, protocol.ServerConnectionMixin) or isinstance(top_layer, protocol.UpstreamConnectLayer):
            return protocol.TlsLayer(top_layer, client_tls, client_tls)

        # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed.
        if isinstance(top_layer, protocol.TlsLayer):
            if isinstance(top_layer.ctx, modes.HttpProxy):
                return protocol.Http1Layer(top_layer, "regular")
            if isinstance(top_layer.ctx, modes.HttpUpstreamProxy):
                return protocol.Http1Layer(top_layer, "upstream")

        # 4. Check for other TLS cases (e.g. after CONNECT).
        if client_tls:
            return protocol.TlsLayer(top_layer, True, True)

        # 4. Check for --tcp
        if self.config.check_tcp(top_layer.server_conn.address):
            return protocol.RawTCPLayer(top_layer)

        # 5. Check for TLS ALPN (HTTP1/HTTP2)
        if isinstance(top_layer, protocol.TlsLayer):
            alpn = top_layer.client_conn.get_alpn_proto_negotiated()
            if alpn == b'h2':
                return protocol.Http2Layer(top_layer, 'transparent')
            if alpn == b'http/1.1':
                return protocol.Http1Layer(top_layer, 'transparent')

        # 6. Check for raw tcp mode
        is_ascii = (
            len(d) == 3 and
            # expect A-Za-z
            all(65 <= x <= 90 or 97 <= x <= 122 for x in six.iterbytes(d))
        )
        if self.config.options.rawtcp and not is_ascii:
            return protocol.RawTCPLayer(top_layer)

        # 7. Assume HTTP1 by default
        return protocol.Http1Layer(top_layer, 'transparent')