Exemple #1
0
 def run(self, flow, is_request):
     r = flow.request if is_request else flow.response
     expected_size = http1.expected_http_body_size(
         flow.request, flow.response if not is_request else None)
     if not (0 <= expected_size <= self.max_size):
         # r.stream may already be a callable, which we want to preserve.
         r.stream = r.stream or True
Exemple #2
0
 def run(self, flow, is_request):
     r = flow.request if is_request else flow.response
     expected_size = http1.expected_http_body_size(
         flow.request, flow.response if not is_request else None)
     if not (0 <= expected_size <= self.max_size):
         # r.stream may already be a callable, which we want to preserve.
         r.stream = r.stream or True
Exemple #3
0
 def read_response_body(self, request, response):
     expected_size = http1.expected_http_body_size(request, response)
     return http1.read_body(
         self.server_conn.rfile,
         expected_size,
         self.config.options.body_size_limit
     )
Exemple #4
0
 def read_request_body(self, request):
     expected_size = http1.expected_http_body_size(request)
     return http1.read_body(
         self.client_conn.rfile,
         expected_size,
         self.config.options.body_size_limit
     )
Exemple #5
0
def parse_http_response(ts, recv, request):
    try:
        recv = BytesIO(recv)
        response = http1.read_response_head(recv)
        body_size = http1.expected_http_body_size(request, response)
        response.data.content, malformed = read_http_body(recv, body_size)
        if malformed:
            response.headers["X-Mitmproxy-Malformed-Body"] = "1"
        return response
    except HttpException as e:
        log.warning("{!r} (timestamp: {})".format(e, ts))
Exemple #6
0
def parse_http_request(ts, sent):
    try:
        sent = BytesIO(sent)
        request = http1.read_request_head(sent)
        body_size = http1.expected_http_body_size(request)
        request.data.content, malformed = read_http_body(sent, body_size)
        if malformed:
            request.headers["X-Mitmproxy-Malformed-Body"] = "1"
        return request
    except HttpException as e:
        log.warning("{!r} (timestamp: {})".format(e, ts))
Exemple #7
0
def parse_http_response(ts, recv, request):
    try:
        recv = BytesIO(recv)
        response = http1.read_response_head(recv)
        body_size = http1.expected_http_body_size(request, response)
        response.data.content, malformed = read_http_body(recv, body_size)
        if malformed:
            response.headers["X-Mitmproxy-Malformed-Body"] = "1"
        return response
    except HttpException as e:
        log.warning("{!r} (timestamp: {})".format(e, ts))
Exemple #8
0
def parse_http_request(ts, sent):
    try:
        sent = BytesIO(sent)
        request = http1.read_request_head(sent)
        body_size = http1.expected_http_body_size(request)
        request.data.content, malformed = read_http_body(sent, body_size)
        if malformed:
            request.headers["X-Mitmproxy-Malformed-Body"] = "1"
        return request
    except HttpException as e:
        log.warning("{!r} (timestamp: {})".format(e, ts))
Exemple #9
0
    def check_close_connection(self, flow):
        request_close = http1.connection_close(flow.request.http_version,
                                               flow.request.headers)
        response_close = http1.connection_close(flow.response.http_version,
                                                flow.response.headers)
        read_until_eof = http1.expected_http_body_size(flow.request,
                                                       flow.response) == -1
        close_connection = request_close or response_close or read_until_eof
        if flow.request.first_line_format == "authority" and flow.response.status_code == 200:
            # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
            # Charles Proxy sends a CONNECT response with HTTP/1.0
            # and no Content-Length header

            return False
        return close_connection
Exemple #10
0
 def run(self, f, is_request):
     if self.max_size:
         r = f.request if is_request else f.response
         try:
             expected_size = http1.expected_http_body_size(
                 f.request, f.response if not is_request else None)
         except exceptions.HTTPException:
             f.reply.kill()
             return
         if expected_size and not r.raw_content and not (0 <= expected_size
                                                         <= self.max_size):
             # r.stream may already be a callable, which we want to preserve.
             r.stream = r.stream or True
             # FIXME: make message generic when we add rquest streaming
             ctx.log.info("Streaming response from %s" % f.request.host)
Exemple #11
0
    def check_close_connection(self, flow):
        request_close = http1.connection_close(
            flow.request.http_version,
            flow.request.headers
        )
        response_close = http1.connection_close(
            flow.response.http_version,
            flow.response.headers
        )
        read_until_eof = http1.expected_http_body_size(flow.request, flow.response) == -1
        close_connection = request_close or response_close or read_until_eof
        if flow.request.form_in == "authority" and flow.response.status_code == 200:
            # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
            # Charles Proxy sends a CONNECT response with HTTP/1.0
            # and no Content-Length header

            return False
        return close_connection
Exemple #12
0
 def read_response_body(self, request, response):
     expected_size = http1.expected_http_body_size(request, response)
     return http1.read_body(self.server_conn.rfile, expected_size, self.config.body_size_limit)
Exemple #13
0
 def read_request_body(self, request):
     expected_size = http1.expected_http_body_size(request)
     return http1.read_body(self.client_conn.rfile, expected_size, self.config.body_size_limit)
Exemple #14
0
def pcap2mitm(pcapfile, mitmfile, tlsmaster=None, stream=False):
    try:
        from mitmproxy import models
        from mitmproxy.flow import FlowWriter
        from netlib.exceptions import HttpException
        from netlib.http import http1
    except ImportError:
        log.warning(
            "In order to use this utility it is required to have the "
            "mitmproxy tool installed (`pip install httpreplay[mitmproxy]`)")
        return False

    if tlsmaster:
        tlsmaster = read_tlsmaster(tlsmaster)
    else:
        tlsmaster = {}

    handlers = {
        443: lambda: https_handler(tlsmaster),
        4443: lambda: https_handler(tlsmaster),
        "generic": http_handler,
    }

    reader = PcapReader(pcapfile)
    reader.tcp = TCPPacketStreamer(reader, handlers)
    writer = FlowWriter(mitmfile)

    l = reader.process()
    if not stream:
        # Sort the http/https requests and responses by their timestamp.
        l = sorted(l, key=lambda x: x[1])

    for s, ts, protocol, sent, recv in l:
        if protocol not in ("http", "https"):
            continue

        srcip, srcport, dstip, dstport = s

        client_conn = models.ClientConnection.make_dummy((srcip, srcport))
        client_conn.timestamp_start = ts

        server_conn = models.ServerConnection.make_dummy((dstip, dstport))
        server_conn.timestamp_start = ts

        flow = models.HTTPFlow(client_conn, server_conn)

        try:
            sent = io.BytesIO(sent.raw)
            request = http1.read_request_head(sent)
            body_size = http1.expected_http_body_size(request)
            request.data.content = "".join(
                http1.read_body(sent, body_size, None))
        except HttpException as e:
            log.warning("Error parsing HTTP request: %s", e)
            continue

        flow.request = models.HTTPRequest.wrap(request)
        flow.request.timestamp_start = client_conn.timestamp_start

        flow.request.host = dstip
        flow.request.port = dstport
        flow.request.scheme = protocol

        try:
            recv = io.BytesIO(recv.raw)
            response = http1.read_response_head(recv)
            body_size = http1.expected_http_body_size(request, response)
            response.data.content = "".join(
                http1.read_body(recv, body_size, None))
        except HttpException as e:
            log.warning("Error parsing HTTP response: %s", e)
            # Fall through (?)

        flow.response = models.HTTPResponse.wrap(response)
        flow.response.timestamp_start = server_conn.timestamp_start

        flow.id = str(
            uuid.UUID(bytes=hashlib.md5(
                b"%d%d%s%s" %
                (client_conn.timestamp_start, server_conn.timestamp_start,
                 request.data.content, response.data.content)).digest()))

        writer.add(flow)
    return True