コード例 #1
0
ファイル: contentviews.py プロジェクト: Angelcold/mitmproxy
 def __call__(self, data, **metadata):
     headers = metadata.get("headers", {})
     ctype = headers.get("content-type")
     if data and ctype:
         ct = http.parse_content_type(ctype) if ctype else None
         ct = "%s/%s" % (ct[0], ct[1])
         if ct in content_types_map:
             return content_types_map[ct][0](data, **metadata)
         elif strutils.is_xml(data):
             return get("XML")(data, **metadata)
     if metadata.get("query"):
         return get("Query")(data, **metadata)
     if data and strutils.is_mostly_bin(data):
         return get("Hex")(data)
     if not data:
         return "No content", []
     return get("Raw")(data)
コード例 #2
0
ファイル: contentviews.py プロジェクト: dufferzafar/mitmproxy
 def __call__(self, data, **metadata):
     headers = metadata.get("headers", {})
     ctype = headers.get("content-type")
     if data and ctype:
         ct = http.parse_content_type(ctype) if ctype else None
         ct = "%s/%s" % (ct[0], ct[1])
         if ct in content_types_map:
             return content_types_map[ct][0](data, **metadata)
         elif strutils.is_xml(data):
             return get("XML")(data, **metadata)
     if metadata.get("query"):
         return get("Query")(data, **metadata)
     if data and strutils.is_mostly_bin(data):
         return get("Hex")(data)
     if not data:
         return "No content", []
     return get("Raw")(data)
コード例 #3
0
ファイル: export.py プロジェクト: mkagenius/mitmproxy
def har_format(flow):

    # -1 indicates that these values do not apply to current request
    ssl_time = -1
    connect_time = -1

    if flow.server_conn:
        connect_time = (flow.server_conn.timestamp_tcp_setup -
                        flow.server_conn.timestamp_start)

        if flow.server_conn.timestamp_ssl_setup is not None:
            ssl_time = (flow.server_conn.timestamp_ssl_setup -
                        flow.server_conn.timestamp_tcp_setup)


    # Calculate raw timings from timestamps. DNS timings can not be calculated
    # for lack of a way to measure it. The same goes for HAR blocked.
    # mitmproxy will open a server connection as soon as it receives the host
    # and port from the client connection. So, the time spent waiting is actually
    # spent waiting between request.timestamp_end and response.timestamp_start
    # thus it correlates to HAR wait instead.
    try:
        timings_raw = {
            'send': flow.request.timestamp_end - flow.request.timestamp_start,
            'receive': flow.response.timestamp_end - flow.response.timestamp_start,
            'wait': flow.response.timestamp_start - flow.request.timestamp_end,
            'connect': connect_time,
            'ssl': ssl_time,
        }

        # HAR timings are integers in ms, so we re-encode the raw timings to that format.
        timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])
    except:
        timings = dict()

    # full_time is the sum of all timings.
    # Timings set to -1 will be ignored as per spec.
    full_time = sum(v for v in timings.values() if v > -1)

    started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))

    response_body_size = -1
    response_body_decoded_size = -1
    response_body_compression = -1
    # Response body size and encoding
    if flow.response and flow.response.raw_content:
        response_body_size = len(flow.response.raw_content)
        response_body_decoded_size = len(flow.response.content)
        response_body_compression = response_body_decoded_size - response_body_size
    
    entry = {
        "testResults":{
            "authentication":flow.authentication,
            "authorization": flow.authorization,
            "otp_leak":flow.otp_leak,
            "payu_salt":flow.payu_salt_leak,
        },
        "keyLogs": flow.key_logs,
        "startedDateTime": started_date_time,
        "time": full_time,
        "request": {
            "method": flow.request.method,
            "url": flow.request.url,
            "httpVersion": flow.request.http_version,
            "cookies": format_request_cookies(flow.request.cookies.fields),
            "headers": name_value(flow.request.headers),
            "queryString": name_value(flow.request.query or {}),
            "headersSize": len(str(flow.request.headers)),
            "bodySize": len(flow.request.content),
        },
        
        "cache": {},
        "timings": timings,
    }

    if flow.response:
        entry["response"] = {
            "status": flow.response.status_code,
            "statusText": flow.response.reason,
            "httpVersion": flow.response.http_version,
            "cookies": format_response_cookies(flow.response.cookies.fields),
            "headers": name_value(flow.response.headers),
            "content": {
                "size": response_body_size,
                "compression": response_body_compression,
                "mimeType": flow.response.headers.get('Content-Type', '')
            },
            "redirectURL": flow.response.headers.get('Location', ''),
            "headersSize": len(str(flow.response.headers)),
            "bodySize": response_body_size,
        }
    # Store binay data as base64
    if strutils.is_mostly_bin(flow.response.content):
        b64 = base64.b64encode(flow.response.content)
        entry["response"]["content"]["text"] = b64.decode('ascii')
        entry["response"]["content"]["encoding"] = "base64"
    else:
        entry["response"]["content"]["text"] = flow.response.text

    if flow.request.method in ["POST", "PUT", "PATCH"]:
        entry["request"]["postData"] = {
            "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],
            "text": flow.request.content,
            "params": name_value(flow.request.urlencoded_form)
        }

    if flow.server_conn:
        entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])

    return entry
コード例 #4
0
def test_is_mostly_bin():
    assert not strutils.is_mostly_bin(b"foo\xFF")
    assert strutils.is_mostly_bin(b"foo" + b"\xFF" * 10)
コード例 #5
0
ファイル: test_strutils.py プロジェクト: Angelcold/mitmproxy
def test_is_mostly_bin():
    assert not strutils.is_mostly_bin(b"foo\xFF")
    assert strutils.is_mostly_bin(b"foo" + b"\xFF" * 10)
コード例 #6
0
ファイル: har_dump.py プロジェクト: weloverandom/mitmproxy
def response(flow):
    """
       Called when a server response has been received.
    """

    # -1 indicates that these values do not apply to current request
    ssl_time = -1
    connect_time = -1

    if flow.server_conn and flow.server_conn not in SERVERS_SEEN:
        connect_time = (flow.server_conn.timestamp_tcp_setup -
                        flow.server_conn.timestamp_start)

        if flow.server_conn.timestamp_ssl_setup is not None:
            ssl_time = (flow.server_conn.timestamp_ssl_setup -
                        flow.server_conn.timestamp_tcp_setup)

        SERVERS_SEEN.add(flow.server_conn)

    # Calculate raw timings from timestamps. DNS timings can not be calculated
    # for lack of a way to measure it. The same goes for HAR blocked.
    # mitmproxy will open a server connection as soon as it receives the host
    # and port from the client connection. So, the time spent waiting is actually
    # spent waiting between request.timestamp_end and response.timestamp_start
    # thus it correlates to HAR wait instead.
    timings_raw = {
        'send': flow.request.timestamp_end - flow.request.timestamp_start,
        'receive': flow.response.timestamp_end - flow.response.timestamp_start,
        'wait': flow.response.timestamp_start - flow.request.timestamp_end,
        'connect': connect_time,
        'ssl': ssl_time,
    }

    # HAR timings are integers in ms, so we re-encode the raw timings to that format.
    timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])

    # full_time is the sum of all timings.
    # Timings set to -1 will be ignored as per spec.
    full_time = sum(v for v in timings.values() if v > -1)

    started_date_time = format_datetime(
        datetime.utcfromtimestamp(flow.request.timestamp_start))

    # Response body size and encoding
    response_body_size = len(flow.response.raw_content)
    response_body_decoded_size = len(flow.response.content)
    response_body_compression = response_body_decoded_size - response_body_size

    entry = {
        "startedDateTime": started_date_time,
        "time": full_time,
        "request": {
            "method": flow.request.method,
            "url": flow.request.url,
            "httpVersion": flow.request.http_version,
            "cookies": format_request_cookies(flow.request.cookies.fields),
            "headers": name_value(flow.request.headers),
            "queryString": name_value(flow.request.query or {}),
            "headersSize": len(str(flow.request.headers)),
            "bodySize": len(flow.request.content),
        },
        "response": {
            "status": flow.response.status_code,
            "statusText": flow.response.reason,
            "httpVersion": flow.response.http_version,
            "cookies": format_response_cookies(flow.response.cookies.fields),
            "headers": name_value(flow.response.headers),
            "content": {
                "size": response_body_size,
                "compression": response_body_compression,
                "mimeType": flow.response.headers.get('Content-Type', '')
            },
            "redirectURL": flow.response.headers.get('Location', ''),
            "headersSize": len(str(flow.response.headers)),
            "bodySize": response_body_size,
        },
        "cache": {},
        "timings": timings,
    }

    # Store binay data as base64
    if strutils.is_mostly_bin(flow.response.content):
        b64 = base64.b64encode(flow.response.content)
        entry["response"]["content"]["text"] = b64.decode('ascii')
        entry["response"]["content"]["encoding"] = "base64"
    else:
        entry["response"]["content"]["text"] = flow.response.text

    if flow.request.method in ["POST", "PUT", "PATCH"]:
        entry["request"]["postData"] = {
            "mimeType": flow.request.headers.get("Content-Type",
                                                 "").split(";")[0],
            "text": flow.request.content,
            "params": name_value(flow.request.urlencoded_form)
        }

    if flow.server_conn:
        entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])

    HAR["log"]["entries"].append(entry)
コード例 #7
0
def response(flow):
    """
       Called when a server response has been received.
    """

    # -1 indicates that these values do not apply to current request
    ssl_time = -1
    connect_time = -1

    if flow.server_conn and flow.server_conn not in SERVERS_SEEN:
        connect_time = (flow.server_conn.timestamp_tcp_setup -
                        flow.server_conn.timestamp_start)

        if flow.server_conn.timestamp_ssl_setup is not None:
            ssl_time = (flow.server_conn.timestamp_ssl_setup -
                        flow.server_conn.timestamp_tcp_setup)

        SERVERS_SEEN.add(flow.server_conn)

    # Calculate raw timings from timestamps. DNS timings can not be calculated
    # for lack of a way to measure it. The same goes for HAR blocked.
    # mitmproxy will open a server connection as soon as it receives the host
    # and port from the client connection. So, the time spent waiting is actually
    # spent waiting between request.timestamp_end and response.timestamp_start
    # thus it correlates to HAR wait instead.
    timings_raw = {
        'send': flow.request.timestamp_end - flow.request.timestamp_start,
        'receive': flow.response.timestamp_end - flow.response.timestamp_start,
        'wait': flow.response.timestamp_start - flow.request.timestamp_end,
        'connect': connect_time,
        'ssl': ssl_time,
    }

    # HAR timings are integers in ms, so we re-encode the raw timings to that format.
    timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])

    # full_time is the sum of all timings.
    # Timings set to -1 will be ignored as per spec.
    full_time = sum(v for v in timings.values() if v > -1)

    started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))

    # Response body size and encoding
    response_body_size = len(flow.response.raw_content)
    try:
        response_body_decoded_size = len(flow.response.content)
    except:
        response_body_decoded_size = len(flow.response.raw_content)
    response_body_compression = response_body_decoded_size - response_body_size
    try:
        request_body_decode_size = len(flow.request.content)
    except:
        request_body_decode_size = len(flow.request.raw_content)

    ip_addr_regex = re.compile(r'\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b')

    try:
        request_headers = name_value(flow.request.headers)
        hostname = ''
        for item in request_headers:
            if item['name'] == 'Host':
                hostname = item['value']
                break
            if item['name'] == ':authority':
                hostname = item['value']
                break 
        if re.search(r':(\d+)', hostname):
            hostname = hostname[:hostname.find(':')]
 
        entry = {
            "startedDateTime": started_date_time,
            "time": full_time,
            "request": {
                "method": flow.request.method,
                "url": re.sub(ip_addr_regex, hostname, flow.request.url) if hostname != '' else flow.request.url,
                "httpVersion": flow.request.http_version,
                "cookies": format_request_cookies(flow.request.cookies.fields),
                "headers": name_value(flow.request.headers),
                "queryString": name_value(flow.request.query or {}),
                "headersSize": len(str(flow.request.headers)),
                "bodySize": request_body_decode_size,
            },
            "response": {
                "status": flow.response.status_code,
                "statusText": flow.response.reason,
                "httpVersion": flow.response.http_version,
                "cookies": format_response_cookies(flow.response.cookies.fields),
                "headers": name_value(flow.response.headers),
                "content": {
                    "size": response_body_size,
                    "compression": response_body_compression,
                    "mimeType": flow.response.headers.get('Content-Type', '')
                },
                "redirectURL": flow.response.headers.get('Location', ''),
                "headersSize": len(str(flow.response.headers)),
                "bodySize": response_body_size,
            },
            "cache": {},
            "timings": timings,
        }

        # Store binary data as base64
        is_mostly_bin = False
        try:
            is_mostly_bin = strutils.is_mostly_bin(flow.response.content)
        except:
            pass
        if is_mostly_bin:
            entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode()
            entry["response"]["content"]["encoding"] = "base64"
        else:
            entry["response"]["content"]["text"] = flow.response.get_text(strict=False)

        if flow.request.method in ["POST", "PUT", "PATCH"]:
            params = [
                {"name": a, "value": b}
                for a, b in flow.request.urlencoded_form.items(multi=True)
            ]
            entry["request"]["postData"] = {
                "mimeType": flow.request.headers.get("Content-Type", ""),
                "text": flow.request.get_text(strict=False),
                "params": params
            }

        try:
            if flow.server_conn.connected():
                entry["serverIPAddress"] = str(flow.server_conn.ip_address[0])
        except:
            pass

        # bypass image files transmission
        if ("mime" in entry["response"]["content"] and "image" in entry["response"]["content"]["mime"]):
            pass
        elif ("mimeType" in entry["response"]["content"] and "image" in entry["response"]["content"]["mimeType"]):
            pass
        else:
            HAR["log"]["entries"].append(entry)
    except:
        logging.exception('Got exception in url: {}'.format(flow.request.url))