def __call__(self, data, **metadata): headers = metadata.get("headers", {}) ctype = headers.get("content-type") if data and ctype: ct = http.parse_content_type(ctype) if ctype else None ct = "%s/%s" % (ct[0], ct[1]) if ct in content_types_map: return content_types_map[ct][0](data, **metadata) elif strutils.is_xml(data): return get("XML")(data, **metadata) if metadata.get("query"): return get("Query")(data, **metadata) if data and strutils.is_mostly_bin(data): return get("Hex")(data) if not data: return "No content", [] return get("Raw")(data)
def har_format(flow): # -1 indicates that these values do not apply to current request ssl_time = -1 connect_time = -1 if flow.server_conn: connect_time = (flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start) if flow.server_conn.timestamp_ssl_setup is not None: ssl_time = (flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup) # Calculate raw timings from timestamps. DNS timings can not be calculated # for lack of a way to measure it. The same goes for HAR blocked. # mitmproxy will open a server connection as soon as it receives the host # and port from the client connection. So, the time spent waiting is actually # spent waiting between request.timestamp_end and response.timestamp_start # thus it correlates to HAR wait instead. try: timings_raw = { 'send': flow.request.timestamp_end - flow.request.timestamp_start, 'receive': flow.response.timestamp_end - flow.response.timestamp_start, 'wait': flow.response.timestamp_start - flow.request.timestamp_end, 'connect': connect_time, 'ssl': ssl_time, } # HAR timings are integers in ms, so we re-encode the raw timings to that format. timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()]) except: timings = dict() # full_time is the sum of all timings. # Timings set to -1 will be ignored as per spec. full_time = sum(v for v in timings.values() if v > -1) started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start)) response_body_size = -1 response_body_decoded_size = -1 response_body_compression = -1 # Response body size and encoding if flow.response and flow.response.raw_content: response_body_size = len(flow.response.raw_content) response_body_decoded_size = len(flow.response.content) response_body_compression = response_body_decoded_size - response_body_size entry = { "testResults":{ "authentication":flow.authentication, "authorization": flow.authorization, "otp_leak":flow.otp_leak, "payu_salt":flow.payu_salt_leak, }, "keyLogs": flow.key_logs, "startedDateTime": started_date_time, "time": full_time, "request": { "method": flow.request.method, "url": flow.request.url, "httpVersion": flow.request.http_version, "cookies": format_request_cookies(flow.request.cookies.fields), "headers": name_value(flow.request.headers), "queryString": name_value(flow.request.query or {}), "headersSize": len(str(flow.request.headers)), "bodySize": len(flow.request.content), }, "cache": {}, "timings": timings, } if flow.response: entry["response"] = { "status": flow.response.status_code, "statusText": flow.response.reason, "httpVersion": flow.response.http_version, "cookies": format_response_cookies(flow.response.cookies.fields), "headers": name_value(flow.response.headers), "content": { "size": response_body_size, "compression": response_body_compression, "mimeType": flow.response.headers.get('Content-Type', '') }, "redirectURL": flow.response.headers.get('Location', ''), "headersSize": len(str(flow.response.headers)), "bodySize": response_body_size, } # Store binay data as base64 if strutils.is_mostly_bin(flow.response.content): b64 = base64.b64encode(flow.response.content) entry["response"]["content"]["text"] = b64.decode('ascii') entry["response"]["content"]["encoding"] = "base64" else: entry["response"]["content"]["text"] = flow.response.text if flow.request.method in ["POST", "PUT", "PATCH"]: entry["request"]["postData"] = { "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0], "text": flow.request.content, "params": name_value(flow.request.urlencoded_form) } if flow.server_conn: entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0]) return entry
def test_is_mostly_bin(): assert not strutils.is_mostly_bin(b"foo\xFF") assert strutils.is_mostly_bin(b"foo" + b"\xFF" * 10)
def response(flow): """ Called when a server response has been received. """ # -1 indicates that these values do not apply to current request ssl_time = -1 connect_time = -1 if flow.server_conn and flow.server_conn not in SERVERS_SEEN: connect_time = (flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start) if flow.server_conn.timestamp_ssl_setup is not None: ssl_time = (flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup) SERVERS_SEEN.add(flow.server_conn) # Calculate raw timings from timestamps. DNS timings can not be calculated # for lack of a way to measure it. The same goes for HAR blocked. # mitmproxy will open a server connection as soon as it receives the host # and port from the client connection. So, the time spent waiting is actually # spent waiting between request.timestamp_end and response.timestamp_start # thus it correlates to HAR wait instead. timings_raw = { 'send': flow.request.timestamp_end - flow.request.timestamp_start, 'receive': flow.response.timestamp_end - flow.response.timestamp_start, 'wait': flow.response.timestamp_start - flow.request.timestamp_end, 'connect': connect_time, 'ssl': ssl_time, } # HAR timings are integers in ms, so we re-encode the raw timings to that format. timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()]) # full_time is the sum of all timings. # Timings set to -1 will be ignored as per spec. full_time = sum(v for v in timings.values() if v > -1) started_date_time = format_datetime( datetime.utcfromtimestamp(flow.request.timestamp_start)) # Response body size and encoding response_body_size = len(flow.response.raw_content) response_body_decoded_size = len(flow.response.content) response_body_compression = response_body_decoded_size - response_body_size entry = { "startedDateTime": started_date_time, "time": full_time, "request": { "method": flow.request.method, "url": flow.request.url, "httpVersion": flow.request.http_version, "cookies": format_request_cookies(flow.request.cookies.fields), "headers": name_value(flow.request.headers), "queryString": name_value(flow.request.query or {}), "headersSize": len(str(flow.request.headers)), "bodySize": len(flow.request.content), }, "response": { "status": flow.response.status_code, "statusText": flow.response.reason, "httpVersion": flow.response.http_version, "cookies": format_response_cookies(flow.response.cookies.fields), "headers": name_value(flow.response.headers), "content": { "size": response_body_size, "compression": response_body_compression, "mimeType": flow.response.headers.get('Content-Type', '') }, "redirectURL": flow.response.headers.get('Location', ''), "headersSize": len(str(flow.response.headers)), "bodySize": response_body_size, }, "cache": {}, "timings": timings, } # Store binay data as base64 if strutils.is_mostly_bin(flow.response.content): b64 = base64.b64encode(flow.response.content) entry["response"]["content"]["text"] = b64.decode('ascii') entry["response"]["content"]["encoding"] = "base64" else: entry["response"]["content"]["text"] = flow.response.text if flow.request.method in ["POST", "PUT", "PATCH"]: entry["request"]["postData"] = { "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0], "text": flow.request.content, "params": name_value(flow.request.urlencoded_form) } if flow.server_conn: entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0]) HAR["log"]["entries"].append(entry)
def response(flow): """ Called when a server response has been received. """ # -1 indicates that these values do not apply to current request ssl_time = -1 connect_time = -1 if flow.server_conn and flow.server_conn not in SERVERS_SEEN: connect_time = (flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start) if flow.server_conn.timestamp_ssl_setup is not None: ssl_time = (flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup) SERVERS_SEEN.add(flow.server_conn) # Calculate raw timings from timestamps. DNS timings can not be calculated # for lack of a way to measure it. The same goes for HAR blocked. # mitmproxy will open a server connection as soon as it receives the host # and port from the client connection. So, the time spent waiting is actually # spent waiting between request.timestamp_end and response.timestamp_start # thus it correlates to HAR wait instead. timings_raw = { 'send': flow.request.timestamp_end - flow.request.timestamp_start, 'receive': flow.response.timestamp_end - flow.response.timestamp_start, 'wait': flow.response.timestamp_start - flow.request.timestamp_end, 'connect': connect_time, 'ssl': ssl_time, } # HAR timings are integers in ms, so we re-encode the raw timings to that format. timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()]) # full_time is the sum of all timings. # Timings set to -1 will be ignored as per spec. full_time = sum(v for v in timings.values() if v > -1) started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start)) # Response body size and encoding response_body_size = len(flow.response.raw_content) try: response_body_decoded_size = len(flow.response.content) except: response_body_decoded_size = len(flow.response.raw_content) response_body_compression = response_body_decoded_size - response_body_size try: request_body_decode_size = len(flow.request.content) except: request_body_decode_size = len(flow.request.raw_content) ip_addr_regex = re.compile(r'\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b') try: request_headers = name_value(flow.request.headers) hostname = '' for item in request_headers: if item['name'] == 'Host': hostname = item['value'] break if item['name'] == ':authority': hostname = item['value'] break if re.search(r':(\d+)', hostname): hostname = hostname[:hostname.find(':')] entry = { "startedDateTime": started_date_time, "time": full_time, "request": { "method": flow.request.method, "url": re.sub(ip_addr_regex, hostname, flow.request.url) if hostname != '' else flow.request.url, "httpVersion": flow.request.http_version, "cookies": format_request_cookies(flow.request.cookies.fields), "headers": name_value(flow.request.headers), "queryString": name_value(flow.request.query or {}), "headersSize": len(str(flow.request.headers)), "bodySize": request_body_decode_size, }, "response": { "status": flow.response.status_code, "statusText": flow.response.reason, "httpVersion": flow.response.http_version, "cookies": format_response_cookies(flow.response.cookies.fields), "headers": name_value(flow.response.headers), "content": { "size": response_body_size, "compression": response_body_compression, "mimeType": flow.response.headers.get('Content-Type', '') }, "redirectURL": flow.response.headers.get('Location', ''), "headersSize": len(str(flow.response.headers)), "bodySize": response_body_size, }, "cache": {}, "timings": timings, } # Store binary data as base64 is_mostly_bin = False try: is_mostly_bin = strutils.is_mostly_bin(flow.response.content) except: pass if is_mostly_bin: entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode() entry["response"]["content"]["encoding"] = "base64" else: entry["response"]["content"]["text"] = flow.response.get_text(strict=False) if flow.request.method in ["POST", "PUT", "PATCH"]: params = [ {"name": a, "value": b} for a, b in flow.request.urlencoded_form.items(multi=True) ] entry["request"]["postData"] = { "mimeType": flow.request.headers.get("Content-Type", ""), "text": flow.request.get_text(strict=False), "params": params } try: if flow.server_conn.connected(): entry["serverIPAddress"] = str(flow.server_conn.ip_address[0]) except: pass # bypass image files transmission if ("mime" in entry["response"]["content"] and "image" in entry["response"]["content"]["mime"]): pass elif ("mimeType" in entry["response"]["content"] and "image" in entry["response"]["content"]["mimeType"]): pass else: HAR["log"]["entries"].append(entry) except: logging.exception('Got exception in url: {}'.format(flow.request.url))