def create_request(self, method, scheme, host, port, path):
"""
this method creates a new artificial and minimalist request also adds it to flowlist
"""
c = models.ClientConnection.make_dummy(("", 0))
s = models.ServerConnection.make_dummy((host, port))
f = models.HTTPFlow(c, s)
headers = models.Headers()
req = models.HTTPRequest("absolute", method, scheme, host, port, path,
b"HTTP/1.1", headers, b"")
f.request = req
self.load_flow(f)
return f
def flow(self):
if any(_ is None for _ in (self._request_timestamp_start,
self._request_timestamp_end,
self._response_timestamp_start,
self._response_timestamp_end,
self._timestamp_ssl_setup)):
s = " ".join("%s=%s" % item for item in dict(
request_start=self._request_timestamp_start,
request_end=self._request_timestamp_end,
response_start=self._response_timestamp_start,
response_end=self._response_timestamp_end,
ssl_setup=self._timestamp_ssl_setup).iteritems())
log.error(s)
client_conn = models.ClientConnection.from_state(
dict(address=dict(address=self._src_address, use_ipv6=False),
clientcert=None,
ssl_established=True,
timestamp_start=self._request_timestamp_start,
timestamp_end=self._request_timestamp_end,
timestamp_ssl_setup=self._timestamp_ssl_setup))
server_conn = models.ServerConnection.from_state(
dict(
ip_address=dict(address=self._src_address, use_ipv6=False),
address=dict(address=self._dst_address, use_ipv6=False),
cert=None,
sni=None,
source_address=dict(address=self._src_address, use_ipv6=False),
#peer_address=dict(address=self._src_address,
# use_ipv6=False),
ssl_established=True,
timestamp_start=self._response_timestamp_start,
timestamp_tcp_setup=self._timestamp_ssl_setup,
timestamp_ssl_setup=self._timestamp_ssl_setup,
timestamp_end=self._response_timestamp_end,
via=None))
flow = models.HTTPFlow(client_conn, server_conn)
flow.request = self.request
flow.response = self.response
return flow
def __call__(self):
if self.mode == "transparent":
self.__initial_server_tls = self.server_tls
self.__initial_server_conn = self.server_conn
while True:
try:
request = self.get_request_from_client()
self.log("request", "debug", [repr(request)])
# Handle Proxy Authentication
# Proxy Authentication conceptually does not work in transparent mode.
# We catch this misconfiguration on startup. Here, we sort out requests
# after a successful CONNECT request (which do not need to be validated anymore)
if self.mode != "transparent" and not self.authenticate(
request):
return
# Make sure that the incoming request matches our expectations
self.validate_request(request)
# Regular Proxy Mode: Handle CONNECT
if self.mode == "regular" and request.first_line_format == "authority":
self.handle_regular_mode_connect(request)
return
except netlib.exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen before/between requests.
return
except netlib.exceptions.NetlibException as e:
self.send_error_response(400, repr(e))
six.reraise(
exceptions.ProtocolException,
exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)),
sys.exc_info()[2])
try:
flow = models.HTTPFlow(self.client_conn,
self.server_conn,
live=self)
flow.request = request
# set upstream auth
if self.mode == "upstream" and self.config.upstream_auth is not None:
flow.request.headers[
"Proxy-Authorization"] = self.config.upstream_auth
self.process_request_hook(flow)
if not flow.response:
self.establish_server_connection(flow)
self.get_response_from_server(flow)
else:
# response was set by an inline script.
# we now need to emulate the responseheaders hook.
flow = self.channel.ask("responseheaders", flow)
self.log("response", "debug", [repr(flow.response)])
flow = self.channel.ask("response", flow)
self.send_response_to_client(flow)
if self.check_close_connection(flow):
return
# Handle 101 Switching Protocols
# It may be useful to pass additional args (such as the upgrade header)
# to next_layer in the future
if flow.response.status_code == 101:
layer = self.ctx.next_layer(self)
layer()
return
# Upstream Proxy Mode: Handle CONNECT
if flow.request.first_line_format == "authority" and flow.response.status_code == 200:
self.handle_upstream_mode_connect(flow.request.copy())
return
except (exceptions.ProtocolException,
netlib.exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
if not flow.response:
flow.error = models.Error(str(e))
self.channel.ask("error", flow)
self.log(traceback.format_exc(), "debug")
return
else:
six.reraise(
exceptions.ProtocolException,
exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)),
sys.exc_info()[2])
finally:
if flow:
flow.live = False
def pcap2mitm(pcapfile, mitmfile, tlsmaster=None, stream=False):
try:
from mitmproxy import models
from mitmproxy.flow import FlowWriter
from netlib.exceptions import HttpException
from netlib.http import http1
except ImportError:
log.warning(
"In order to use this utility it is required to have the "
"mitmproxy tool installed (`pip install httpreplay[mitmproxy]`)")
return False
if tlsmaster:
tlsmaster = read_tlsmaster(tlsmaster)
else:
tlsmaster = {}
handlers = {
443: lambda: https_handler(tlsmaster),
4443: lambda: https_handler(tlsmaster),
"generic": http_handler,
}
reader = PcapReader(pcapfile)
reader.tcp = TCPPacketStreamer(reader, handlers)
writer = FlowWriter(mitmfile)
l = reader.process()
if not stream:
# Sort the http/https requests and responses by their timestamp.
l = sorted(l, key=lambda x: x[1])
for s, ts, protocol, sent, recv in l:
if protocol not in ("http", "https"):
continue
srcip, srcport, dstip, dstport = s
client_conn = models.ClientConnection.make_dummy((srcip, srcport))
client_conn.timestamp_start = ts
server_conn = models.ServerConnection.make_dummy((dstip, dstport))
server_conn.timestamp_start = ts
flow = models.HTTPFlow(client_conn, server_conn)
try:
sent = io.BytesIO(sent.raw)
request = http1.read_request_head(sent)
body_size = http1.expected_http_body_size(request)
request.data.content = "".join(
http1.read_body(sent, body_size, None))
except HttpException as e:
log.warning("Error parsing HTTP request: %s", e)
continue
flow.request = models.HTTPRequest.wrap(request)
flow.request.timestamp_start = client_conn.timestamp_start
flow.request.host = dstip
flow.request.port = dstport
flow.request.scheme = protocol
try:
recv = io.BytesIO(recv.raw)
response = http1.read_response_head(recv)
body_size = http1.expected_http_body_size(request, response)
response.data.content = "".join(
http1.read_body(recv, body_size, None))
except HttpException as e:
log.warning("Error parsing HTTP response: %s", e)
# Fall through (?)
flow.response = models.HTTPResponse.wrap(response)
flow.response.timestamp_start = server_conn.timestamp_start
flow.id = str(
uuid.UUID(bytes=hashlib.md5(
b"%d%d%s%s" %
(client_conn.timestamp_start, server_conn.timestamp_start,
request.data.content, response.data.content)).digest()))
writer.add(flow)
return True
def __call__(self):
if self.mode == "transparent":
self.__initial_server_tls = self.server_tls
self.__initial_server_conn = self.server_conn
while True:
try:
request = self.get_request_from_client()
# Make sure that the incoming request matches our expectations
self.validate_request(request)
except netlib.exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen before/between requests.
return
except netlib.exceptions.HttpException as e:
# We optimistically guess there might be an HTTP client on the
# other end
self.send_error_response(400, repr(e))
six.reraise(
exceptions.ProtocolException,
exceptions.ProtocolException(
"HTTP protocol error in client request: {}".format(e)
),
sys.exc_info()[2]
)
self.log("request", "debug", [repr(request)])
# Handle Proxy Authentication
# Proxy Authentication conceptually does not work in transparent mode.
# We catch this misconfiguration on startup. Here, we sort out requests
# after a successful CONNECT request (which do not need to be validated anymore)
if not (self.http_authenticated or self.authenticate(request)):
return
flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self)
flow.request = request
try:
# Regular Proxy Mode: Handle CONNECT
if self.mode == "regular" and request.first_line_format == "authority":
self.handle_regular_mode_connect(request)
return
except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e:
# HTTPS tasting means that ordinary errors like resolution and
# connection errors can happen here.
self.send_error_response(502, repr(e))
flow.error = models.Error(str(e))
self.channel.ask("error", flow)
return
# update host header in reverse proxy mode
if self.config.options.mode == "reverse":
if six.PY2:
flow.request.headers["Host"] = self.config.upstream_server.address.host.encode()
else:
flow.request.headers["Host"] = self.config.upstream_server.address.host
# set upstream auth
if self.mode == "upstream" and self.config.upstream_auth is not None:
flow.request.headers["Proxy-Authorization"] = self.config.upstream_auth
self.process_request_hook(flow)
try:
if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
# We only support RFC6455 with WebSockets version 13
# allow inline scripts to manipulate the client handshake
self.channel.ask("websockets_handshake", flow)
if not flow.response:
self.establish_server_connection(
flow.request.host,
flow.request.port,
flow.request.scheme
)
self.get_response_from_server(flow)
else:
# response was set by an inline script.
# we now need to emulate the responseheaders hook.
flow = self.channel.ask("responseheaders", flow)
self.log("response", "debug", [repr(flow.response)])
flow = self.channel.ask("response", flow)
self.send_response_to_client(flow)
if self.check_close_connection(flow):
return
# Handle 101 Switching Protocols
if flow.response.status_code == 101:
return self.handle_101_switching_protocols(flow)
# Upstream Proxy Mode: Handle CONNECT
if flow.request.first_line_format == "authority" and flow.response.status_code == 200:
self.handle_upstream_mode_connect(flow.request.copy())
return
except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
if not flow.response:
flow.error = models.Error(str(e))
self.channel.ask("error", flow)
return
else:
six.reraise(exceptions.ProtocolException, exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
finally:
if flow:
flow.live = False
def pcap2mitm(pcapfile, mitmfile, tlsmaster, stream):
try:
from mitmproxy import models
from mitmproxy.flow import FlowWriter
from netlib.http import http1
from netlib.exceptions import HttpException
except ImportError:
raise click.Abort(
"In order to use this utility it is required to have the "
"mitmproxy tool installed (`pip install httpreplay[mitmproxy]`)")
class NetlibHttpProtocol(Protocol):
"""
Like HttpProtocol, but actually covering edge-cases.
"""
@staticmethod
def read_body(io, expected_size):
"""
Read a (malformed) HTTP body.
Returns:
A (body: bytes, is_malformed: bool) tuple.
"""
body_start = io.tell()
try:
content = b"".join(http1.read_body(io, expected_size, None))
if io.read(): # leftover?
raise HttpException()
return content, False
except HttpException:
io.seek(body_start)
return io.read(), True
def parse_request(self, ts, sent):
try:
sent = BytesIO(sent)
request = http1.read_request_head(sent)
body_size = http1.expected_http_body_size(request)
request.data.content, malformed = self.read_body(
sent, body_size)
if malformed:
request.headers["X-Mitmproxy-Malformed-Body"] = "1"
return request
except HttpException as e:
log.warning("{!r} (timestamp: {})".format(e, ts))
def parse_response(self, ts, recv, request):
try:
recv = BytesIO(recv)
response = http1.read_response_head(recv)
body_size = http1.expected_http_body_size(request, response)
response.data.content, malformed = self.read_body(
recv, body_size)
if malformed:
response.headers["X-Mitmproxy-Malformed-Body"] = "1"
return response
except HttpException as e:
log.warning("{!r} (timestamp: {})".format(e, ts))
def handle(self, s, ts, protocol, sent, recv):
if protocol not in ("tcp", "tls"):
self.parent.handle(s, ts, protocol, sent, recv)
return
req = None
if sent:
req = self.parse_request(ts, sent)
protocols = {
"tcp": "http",
"tls": "https",
}
# Only try to decode the HTTP response if the request was valid HTTP.
if req:
res = self.parse_response(ts, recv, req)
# Report this stream as being a valid HTTP stream.
self.parent.handle(s, ts, protocols[protocol], req or sent,
res)
else:
# This wasn't a valid HTTP stream so we forward the original TCP
# or TLS stream straight ahead to our parent.
self.parent.handle(s, ts, protocol, sent, recv)
if tlsmaster:
tlsmaster = read_tlsmaster(tlsmaster)
else:
tlsmaster = {}
netlib_http_handler = lambda: NetlibHttpProtocol()
netlib_https_handler = lambda: TLSStream(NetlibHttpProtocol(), tlsmaster)
handlers = {
443: netlib_https_handler,
4443: netlib_https_handler,
"generic": netlib_http_handler,
}
reader = PcapReader(pcapfile)
reader.tcp = TCPPacketStreamer(reader, handlers)
writer = FlowWriter(mitmfile)
l = reader.process()
if not stream:
# Sort the http/https requests and responses by their timestamp.
l = sorted(l, key=lambda x: x[1])
for addrs, timestamp, protocol, sent, recv in l:
if protocol not in ("http", "https"):
continue
srcip, srcport, dstip, dstport = addrs
client_conn = models.ClientConnection.make_dummy((srcip, srcport))
client_conn.timestamp_start = timestamp
server_conn = models.ServerConnection.make_dummy((dstip, dstport))
server_conn.timestamp_start = timestamp
flow = models.HTTPFlow(client_conn, server_conn)
flow.request = models.HTTPRequest.wrap(sent)
flow.request.host, flow.request.port = dstip, dstport
flow.request.scheme = protocol
if recv:
flow.response = models.HTTPResponse.wrap(recv)
writer.add(flow)
