def run(self, flow, is_request):
r = flow.request if is_request else flow.response
expected_size = http1.expected_http_body_size(
flow.request, flow.response if not is_request else None)
if not (0 <= expected_size <= self.max_size):
# r.stream may already be a callable, which we want to preserve.
r.stream = r.stream or True
def run(self, flow, is_request):
r = flow.request if is_request else flow.response
expected_size = http1.expected_http_body_size(
flow.request, flow.response if not is_request else None)
if not (0 <= expected_size <= self.max_size):
# r.stream may already be a callable, which we want to preserve.
r.stream = r.stream or True
def read_response_body(self, request, response):
expected_size = http1.expected_http_body_size(request, response)
return http1.read_body(
self.server_conn.rfile,
expected_size,
self.config.options.body_size_limit
)
def read_request_body(self, request):
expected_size = http1.expected_http_body_size(request)
return http1.read_body(
self.client_conn.rfile,
expected_size,
self.config.options.body_size_limit
)
def parse_http_response(ts, recv, request):
try:
recv = BytesIO(recv)
response = http1.read_response_head(recv)
body_size = http1.expected_http_body_size(request, response)
response.data.content, malformed = read_http_body(recv, body_size)
if malformed:
response.headers["X-Mitmproxy-Malformed-Body"] = "1"
return response
except HttpException as e:
log.warning("{!r} (timestamp: {})".format(e, ts))
def parse_http_request(ts, sent):
try:
sent = BytesIO(sent)
request = http1.read_request_head(sent)
body_size = http1.expected_http_body_size(request)
request.data.content, malformed = read_http_body(sent, body_size)
if malformed:
request.headers["X-Mitmproxy-Malformed-Body"] = "1"
return request
except HttpException as e:
log.warning("{!r} (timestamp: {})".format(e, ts))
def parse_http_response(ts, recv, request):
try:
recv = BytesIO(recv)
response = http1.read_response_head(recv)
body_size = http1.expected_http_body_size(request, response)
response.data.content, malformed = read_http_body(recv, body_size)
if malformed:
response.headers["X-Mitmproxy-Malformed-Body"] = "1"
return response
except HttpException as e:
log.warning("{!r} (timestamp: {})".format(e, ts))
def parse_http_request(ts, sent):
try:
sent = BytesIO(sent)
request = http1.read_request_head(sent)
body_size = http1.expected_http_body_size(request)
request.data.content, malformed = read_http_body(sent, body_size)
if malformed:
request.headers["X-Mitmproxy-Malformed-Body"] = "1"
return request
except HttpException as e:
log.warning("{!r} (timestamp: {})".format(e, ts))
def check_close_connection(self, flow):
request_close = http1.connection_close(flow.request.http_version,
flow.request.headers)
response_close = http1.connection_close(flow.response.http_version,
flow.response.headers)
read_until_eof = http1.expected_http_body_size(flow.request,
flow.response) == -1
close_connection = request_close or response_close or read_until_eof
if flow.request.first_line_format == "authority" and flow.response.status_code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Charles Proxy sends a CONNECT response with HTTP/1.0
# and no Content-Length header
return False
return close_connection
def run(self, f, is_request):
if self.max_size:
r = f.request if is_request else f.response
try:
expected_size = http1.expected_http_body_size(
f.request, f.response if not is_request else None)
except exceptions.HTTPException:
f.reply.kill()
return
if expected_size and not r.raw_content and not (0 <= expected_size
<= self.max_size):
# r.stream may already be a callable, which we want to preserve.
r.stream = r.stream or True
# FIXME: make message generic when we add rquest streaming
ctx.log.info("Streaming response from %s" % f.request.host)
def check_close_connection(self, flow):
request_close = http1.connection_close(
flow.request.http_version,
flow.request.headers
)
response_close = http1.connection_close(
flow.response.http_version,
flow.response.headers
)
read_until_eof = http1.expected_http_body_size(flow.request, flow.response) == -1
close_connection = request_close or response_close or read_until_eof
if flow.request.form_in == "authority" and flow.response.status_code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Charles Proxy sends a CONNECT response with HTTP/1.0
# and no Content-Length header
return False
return close_connection
def read_response_body(self, request, response):
expected_size = http1.expected_http_body_size(request, response)
return http1.read_body(self.server_conn.rfile, expected_size, self.config.body_size_limit)
def read_request_body(self, request):
expected_size = http1.expected_http_body_size(request)
return http1.read_body(self.client_conn.rfile, expected_size, self.config.body_size_limit)
def pcap2mitm(pcapfile, mitmfile, tlsmaster=None, stream=False):
try:
from mitmproxy import models
from mitmproxy.flow import FlowWriter
from netlib.exceptions import HttpException
from netlib.http import http1
except ImportError:
log.warning(
"In order to use this utility it is required to have the "
"mitmproxy tool installed (`pip install httpreplay[mitmproxy]`)")
return False
if tlsmaster:
tlsmaster = read_tlsmaster(tlsmaster)
else:
tlsmaster = {}
handlers = {
443: lambda: https_handler(tlsmaster),
4443: lambda: https_handler(tlsmaster),
"generic": http_handler,
}
reader = PcapReader(pcapfile)
reader.tcp = TCPPacketStreamer(reader, handlers)
writer = FlowWriter(mitmfile)
l = reader.process()
if not stream:
# Sort the http/https requests and responses by their timestamp.
l = sorted(l, key=lambda x: x[1])
for s, ts, protocol, sent, recv in l:
if protocol not in ("http", "https"):
continue
srcip, srcport, dstip, dstport = s
client_conn = models.ClientConnection.make_dummy((srcip, srcport))
client_conn.timestamp_start = ts
server_conn = models.ServerConnection.make_dummy((dstip, dstport))
server_conn.timestamp_start = ts
flow = models.HTTPFlow(client_conn, server_conn)
try:
sent = io.BytesIO(sent.raw)
request = http1.read_request_head(sent)
body_size = http1.expected_http_body_size(request)
request.data.content = "".join(
http1.read_body(sent, body_size, None))
except HttpException as e:
log.warning("Error parsing HTTP request: %s", e)
continue
flow.request = models.HTTPRequest.wrap(request)
flow.request.timestamp_start = client_conn.timestamp_start
flow.request.host = dstip
flow.request.port = dstport
flow.request.scheme = protocol
try:
recv = io.BytesIO(recv.raw)
response = http1.read_response_head(recv)
body_size = http1.expected_http_body_size(request, response)
response.data.content = "".join(
http1.read_body(recv, body_size, None))
except HttpException as e:
log.warning("Error parsing HTTP response: %s", e)
# Fall through (?)
flow.response = models.HTTPResponse.wrap(response)
flow.response.timestamp_start = server_conn.timestamp_start
flow.id = str(
uuid.UUID(bytes=hashlib.md5(
b"%d%d%s%s" %
(client_conn.timestamp_start, server_conn.timestamp_start,
request.data.content, response.data.content)).digest()))
writer.add(flow)
return True