Esempio n. 1
0
    def handle_flow(self):
        flow = HTTPFlow(self.c.client_conn, self.c.server_conn,
                        self.change_server)
        try:
            req = HTTPRequest.from_stream(
                self.c.client_conn.rfile,
                body_size_limit=self.c.config.body_size_limit)
            self.c.log("request", "debug",
                       [req._assemble_first_line(req.form_in)])
            send_upstream = self.process_request(flow, req)
            if not send_upstream:
                return True

            # Be careful NOT to assign the request to the flow before
            # process_request completes. This is because the call can raise an
            # exception. If the request object is already attached, this results
            # in an Error object that has an attached request that has not been
            # sent through to the Master.
            flow.request = req
            request_reply = self.c.channel.ask("request", flow.request)
            flow.server_conn = self.c.server_conn

            if request_reply is None or request_reply == KILL:
                return False

            if isinstance(request_reply, HTTPResponse):
                flow.response = request_reply
            else:
                flow.response = self.get_response_from_server(flow.request)

            flow.server_conn = self.c.server_conn  # no further manipulation of self.c.server_conn beyond this point
            # we can safely set it as the final attribute value here.

            self.c.log("response", "debug",
                       [flow.response._assemble_first_line()])
            response_reply = self.c.channel.ask("response", flow.response)
            if response_reply is None or response_reply == KILL:
                return False

            self.c.client_conn.send(flow.response._assemble())
            flow.timestamp_end = utils.timestamp()

            if (http.connection_close(flow.request.httpversion,
                                      flow.request.headers)
                    or http.connection_close(flow.response.httpversion,
                                             flow.response.headers)):
                return False

            if flow.request.form_in == "authority":
                self.ssl_upgrade()

            # If the user has changed the target server on this connection,
            # restore the original target server
            self.restore_server()
            return True
        except (HttpAuthenticationError, http.HttpError, proxy.ProxyError,
                tcp.NetLibError), e:
            self.handle_error(e, flow)
Esempio n. 2
0
    def handle_flow(self):
        flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.change_server)
        try:
            req = HTTPRequest.from_stream(self.c.client_conn.rfile,
                                          body_size_limit=self.c.config.body_size_limit)
            self.c.log("request", "debug", [req._assemble_first_line(req.form_in)])
            send_request_upstream = self.process_request(flow, req)
            if not send_request_upstream:
                return True

            # Be careful NOT to assign the request to the flow before
            # process_request completes. This is because the call can raise an
            # exception. If the request object is already attached, this results
            # in an Error object that has an attached request that has not been
            # sent through to the Master.
            flow.request = req
            request_reply = self.c.channel.ask("request", flow.request)
            flow.server_conn = self.c.server_conn

            if request_reply is None or request_reply == KILL:
                return False

            if isinstance(request_reply, HTTPResponse):
                flow.response = request_reply
            else:
                flow.response = self.get_response_from_server(flow.request)

            flow.server_conn = self.c.server_conn  # no further manipulation of self.c.server_conn beyond this point
            # we can safely set it as the final attribute value here.

            self.c.log("response", "debug", [flow.response._assemble_first_line()])
            response_reply = self.c.channel.ask("response", flow.response)
            if response_reply is None or response_reply == KILL:
                return False

            self.c.client_conn.send(flow.response._assemble())
            flow.timestamp_end = utils.timestamp()

            if (http.connection_close(flow.request.httpversion, flow.request.headers) or
                    http.connection_close(flow.response.httpversion, flow.response.headers)):
                if flow.request.form_in == "authority" and flow.response.code == 200:
                    # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
                    # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
                    pass
                else:
                    return False

            if flow.request.form_in == "authority" and flow.response.code == 200:
                self.ssl_upgrade()

            # If the user has changed the target server on this connection,
            # restore the original target server
            self.restore_server()
            return True
        except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e:
            self.handle_error(e, flow)
Esempio n. 3
0
def test_connection_close():
    h = odict.ODictCaseless()
    assert http.connection_close((1, 0), h)
    assert not http.connection_close((1, 1), h)

    h["connection"] = ["keep-alive"]
    assert not http.connection_close((1, 1), h)

    h["connection"] = ["close"]
    assert http.connection_close((1, 1), h)
Esempio n. 4
0
def test_connection_close():
    h = odict.ODictCaseless()
    assert http.connection_close((1, 0), h)
    assert not http.connection_close((1, 1), h)

    h["connection"] = ["keep-alive"]
    assert not http.connection_close((1, 1), h)

    h["connection"] = ["close"]
    assert http.connection_close((1, 1), h)
Esempio n. 5
0
 def check_close_connection(self, flow):
     """
     Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so.
     """
     close_connection = (
         http.connection_close(flow.request.httpversion, flow.request.headers) or
         http.connection_close(flow.response.httpversion, flow.response.headers) or
         http.expected_http_body_size(flow.response.headers, False, flow.request.method,
                                      flow.response.code) == -1)
     if close_connection:
         if flow.request.form_in == "authority" and flow.response.code == 200:
             # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
             # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
             pass
         else:
             return True
     return False
Esempio n. 6
0
 def check_close_connection(self, flow):
     """
     Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so.
     """
     close_connection = (http.connection_close(flow.request.httpversion,
                                               flow.request.headers)
                         or http.connection_close(flow.response.httpversion,
                                                  flow.response.headers)
                         or http.expected_http_body_size(
                             flow.response.headers, False,
                             flow.request.method, flow.response.code) == -1)
     if close_connection:
         if flow.request.form_in == "authority" and flow.response.code == 200:
             # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
             # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
             pass
         else:
             return True
     return False
Esempio n. 7
0
    def handle_flow(self):
        flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live)
        try:
            req = HTTPRequest.from_stream(self.c.client_conn.rfile,
                                          body_size_limit=self.c.config.body_size_limit)
            self.c.log("request", "debug", [req._assemble_first_line(req.form_in)])
            send_request_upstream = self.process_request(flow, req)
            if not send_request_upstream:
                return True

            # Be careful NOT to assign the request to the flow before
            # process_request completes. This is because the call can raise an
            # exception. If the request object is already attached, this results
            # in an Error object that has an attached request that has not been
            # sent through to the Master.
            flow.request = req
            request_reply = self.c.channel.ask("request", flow.request)
            self.determine_server_address(flow, flow.request)
            flow.server_conn = self.c.server_conn  # Update server_conn attribute on the flow

            if request_reply is None or request_reply == KILL:
                return False

            if isinstance(request_reply, HTTPResponse):
                flow.response = request_reply
            else:

                # read initially in "stream" mode, so we can get the headers separately
                flow.response = self.get_response_from_server(flow.request, include_body=False)

                # call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True
                self.c.channel.ask("responseheaders", flow.response)

                # now get the rest of the request body, if body still needs to be read but not streaming this response
                if flow.response.stream:
                    flow.response.content = CONTENT_MISSING
                else:
                    flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers,
                                                                self.c.config.body_size_limit,
                                                                flow.request.method, flow.response.code, False)

            # no further manipulation of self.c.server_conn beyond this point
            # we can safely set it as the final attribute value here.
            flow.server_conn = self.c.server_conn

            self.c.log("response", "debug", [flow.response._assemble_first_line()])
            response_reply = self.c.channel.ask("response", flow.response)
            if response_reply is None or response_reply == KILL:
                return False

            if not flow.response.stream:
                # no streaming:
                # we already received the full response from the server and can send it to the client straight away.
                self.c.client_conn.send(flow.response._assemble())
            else:
                # streaming:
                # First send the body and then transfer the response incrementally:
                h = flow.response._assemble_head(preserve_transfer_encoding=True)
                self.c.client_conn.send(h)
                for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
                                                         flow.response.headers,
                                                         self.c.config.body_size_limit, flow.request.method,
                                                         flow.response.code, False, 4096):
                    for part in chunk:
                        self.c.client_conn.wfile.write(part)
                    self.c.client_conn.wfile.flush()
                flow.response.timestamp_end = utils.timestamp()

            flow.timestamp_end = utils.timestamp()

            close_connection = (
                http.connection_close(flow.request.httpversion, flow.request.headers) or
                http.connection_close(flow.response.httpversion, flow.response.headers) or
                http.expected_http_body_size(flow.response.headers, False, flow.request.method,
                                             flow.response.code) == -1)
            if close_connection:
                if flow.request.form_in == "authority" and flow.response.code == 200:
                    # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
                    # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
                    pass
                else:
                    return False

            if flow.request.form_in == "authority" and flow.response.code == 200:
                # TODO: Eventually add headers (space/usefulness tradeoff)
                # Make sure to add state info before the actual upgrade happens.
                # During the upgrade, we may receive an SNI indication from the client,
                # which resets the upstream connection. If this is the case, we must
                # already re-issue the CONNECT request at this point.
                self.c.server_conn.state.append(("http", {"state": "connect",
                                                          "host": flow.request.host,
                                                          "port": flow.request.port}))
                self.ssl_upgrade()

            # If the user has changed the target server on this connection,
            # restore the original target server
            flow.live.restore_server()
            flow.live = None

            return True
        except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e:
            self.handle_error(e, flow)
Esempio n. 8
0
    def handle_request(self, cc):
        try:
            request, err = None, None
            request = self.read_request(cc)
            if request is None:
                return
            cc.requestcount += 1

            request_reply = self.channel.ask("request", request)
            if request_reply is None or request_reply == KILL:
                return
            elif isinstance(request_reply, flow.Response):
                request = False
                response = request_reply
                response_reply = self.channel.ask("response", response)
            else:
                request = request_reply
                if self.config.reverse_proxy:
                    scheme, host, port = self.config.reverse_proxy
                elif self.config.forward_proxy:
                    scheme, host, port = self.config.forward_proxy
                else:
                    scheme, host, port = request.scheme, request.host, request.port

                # If we've already pumped a request over this connection,
                # it's possible that the server has timed out. If this is
                # the case, we want to reconnect without sending an error
                # to the client.
                while 1:
                    sc = self.get_server_connection(cc, scheme, host, port, self.sni, request=request)
                    sc.send(request)
                    if sc.requestcount == 1: # add timestamps only for first request (others are not directly affected)
                        request.tcp_setup_timestamp = sc.tcp_setup_timestamp
                        request.ssl_setup_timestamp = sc.ssl_setup_timestamp
                    sc.rfile.reset_timestamps()
                    try:
                        peername = sc.connection.getpeername()
                        if peername:
                            request.ip = peername[0]
                        httpversion, code, msg, headers, content = http.read_response(
                            sc.rfile,
                            request.method,
                            self.config.body_size_limit
                        )
                    except http.HttpErrorConnClosed:
                        self.del_server_connection()
                        if sc.requestcount > 1:
                            continue
                        else:
                            raise
                    except http.HttpError:
                        raise ProxyError(502, "Invalid server response.")
                    else:
                        break

                response = flow.Response(
                    request, httpversion, code, msg, headers, content, sc.cert,
                    sc.rfile.first_byte_timestamp
                )
                response_reply = self.channel.ask("response", response)
                # Not replying to the server invalidates the server
                # connection, so we terminate.
                if response_reply == KILL:
                    sc.terminate()

            if response_reply == KILL:
                return
            else:
                response = response_reply
                self.send_response(response)
                if request and http.connection_close(request.httpversion, request.headers):
                    return
                # We could keep the client connection when the server
                # connection needs to go away.  However, we want to mimic
                # behaviour as closely as possible to the client, so we
                # disconnect.
                if http.connection_close(response.httpversion, response.headers):
                    return
        except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e:
            if hasattr(e, "code"):
                cc.error = "%s: %s"%(e.code, e.msg)
            else:
                cc.error = str(e)

            if request:
                err = flow.Error(request, cc.error)
                self.channel.ask("error", err)
                self.log(
                    cc, cc.error,
                    ["url: %s"%request.get_url()]
                )
            else:
                self.log(cc, cc.error)
            if isinstance(e, ProxyError):
                self.send_error(e.code, e.msg, e.headers)
Esempio n. 9
0
                    response = flow.Response(request, httpversion, code, msg,
                                             headers, content, sc.cert,
                                             sc.rfile.first_byte_timestamp)
                    response_reply = self.channel.ask(response)
                    # Not replying to the server invalidates the server
                    # connection, so we terminate.
                    if response_reply == KILL:
                        sc.terminate()

                if response_reply == KILL:
                    return
                else:
                    response = response_reply
                    self.send_response(response)
                    if request and http.connection_close(
                            request.httpversion, request.headers):
                        return
                    # We could keep the client connection when the server
                    # connection needs to go away.  However, we want to mimic
                    # behaviour as closely as possible to the client, so we
                    # disconnect.
                    if http.connection_close(response.httpversion,
                                             response.headers):
                        return
        except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e:
            if hasattr(e, "code"):
                cc.error = "%s: %s" % (e.code, e.msg)
            else:
                cc.error = str(e)

            if request:
Esempio n. 10
0
                    response = flow.Response(
                        request, httpversion, code, msg, headers, content, sc.cert,
                        sc.rfile.first_byte_timestamp
                    )
                    response_reply = self.channel.ask(response)
                    # Not replying to the server invalidates the server
                    # connection, so we terminate.
                    if response_reply == KILL:
                        sc.terminate()

                if response_reply == KILL:
                    return
                else:
                    response = response_reply
                    self.send_response(response)
                    if request and http.connection_close(request.httpversion, request.headers):
                        return
                    # We could keep the client connection when the server
                    # connection needs to go away.  However, we want to mimic
                    # behaviour as closely as possible to the client, so we
                    # disconnect.
                    if http.connection_close(response.httpversion, response.headers):
                        return
        except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e:
            if hasattr(e, "code"):
                cc.error = "%s: %s"%(e.code, e.msg)
            else:
                cc.error = str(e)

            if request:
                err = flow.Error(request, cc.error)
Esempio n. 11
0
    def handle_flow(self):
        flow = HTTPFlow(self.c.client_conn, self.c.server_conn,
                        self.change_server)
        try:
            req = HTTPRequest.from_stream(
                self.c.client_conn.rfile,
                body_size_limit=self.c.config.body_size_limit)
            self.c.log("request", "debug",
                       [req._assemble_first_line(req.form_in)])
            send_request_upstream = self.process_request(flow, req)
            if not send_request_upstream:
                return True

            # Be careful NOT to assign the request to the flow before
            # process_request completes. This is because the call can raise an
            # exception. If the request object is already attached, this results
            # in an Error object that has an attached request that has not been
            # sent through to the Master.
            flow.request = req
            request_reply = self.c.channel.ask("request", flow.request)
            self.determine_server_address(flow, flow.request)
            flow.server_conn = self.c.server_conn  # Update server_conn attribute on the flow

            if request_reply is None or request_reply == KILL:
                return False

            if isinstance(request_reply, HTTPResponse):
                flow.response = request_reply
            else:

                # read initially in "stream" mode, so we can get the headers separately
                flow.response = self.get_response_from_server(
                    flow.request, include_body=False)

                # call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True
                self.c.channel.ask("responseheaders", flow.response)

                # now get the rest of the request body, if body still needs to be read but not streaming this response
                if flow.response.stream:
                    flow.response.content = CONTENT_MISSING
                else:
                    flow.response.content = http.read_http_body(
                        self.c.server_conn.rfile, flow.response.headers,
                        self.c.config.body_size_limit, flow.request.method,
                        flow.response.code, False)

            # no further manipulation of self.c.server_conn beyond this point
            # we can safely set it as the final attribute value here.
            flow.server_conn = self.c.server_conn

            self.c.log("response", "debug",
                       [flow.response._assemble_first_line()])
            response_reply = self.c.channel.ask("response", flow.response)
            if response_reply is None or response_reply == KILL:
                return False

            if not flow.response.stream:
                # no streaming:
                # we already received the full response from the server and can send it to the client straight away.
                self.c.client_conn.send(flow.response._assemble())
            else:
                # streaming:
                # First send the body and then transfer the response incrementally:
                h = flow.response._assemble_head(
                    preserve_transfer_encoding=True)
                self.c.client_conn.send(h)
                for chunk in http.read_http_body_chunked(
                        self.c.server_conn.rfile, flow.response.headers,
                        self.c.config.body_size_limit, flow.request.method,
                        flow.response.code, False, 4096):
                    for part in chunk:
                        self.c.client_conn.wfile.write(part)
                    self.c.client_conn.wfile.flush()

            flow.timestamp_end = utils.timestamp()

            close_connection = (
                http.connection_close(flow.request.httpversion,
                                      flow.request.headers)
                or http.connection_close(flow.response.httpversion,
                                         flow.response.headers)
                or http.expected_http_body_size(flow.response.headers, False,
                                                flow.request.method,
                                                flow.response.code) == -1)
            if close_connection:
                if flow.request.form_in == "authority" and flow.response.code == 200:
                    # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
                    # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
                    pass
                else:
                    return False

            if flow.request.form_in == "authority" and flow.response.code == 200:
                # TODO: Eventually add headers (space/usefulness tradeoff)
                # Make sure to add state info before the actual upgrade happens.
                # During the upgrade, we may receive an SNI indication from the client,
                # which resets the upstream connection. If this is the case, we must
                # already re-issue the CONNECT request at this point.
                self.c.server_conn.state.append(("http", {
                    "state": "connect",
                    "host": flow.request.host,
                    "port": flow.request.port
                }))
                self.ssl_upgrade()

            # If the user has changed the target server on this connection,
            # restore the original target server
            self.restore_server()

            return True
        except (HttpAuthenticationError, http.HttpError, proxy.ProxyError,
                tcp.NetLibError), e:
            self.handle_error(e, flow)