def setup_connection(self):
self.client = mitmproxy.net.tcp.TCPClient(("127.0.0.1", self.proxy.port))
self.client.connect()
# send CONNECT request
self.client.wfile.write(http1.assemble_request(mitmproxy.net.http.Request(
'authority',
b'CONNECT',
b'',
b'localhost',
self.server.server.address[1],
b'/',
b'HTTP/1.1',
[(b'host', b'localhost:%d' % self.server.server.address[1])],
b'',
)))
self.client.wfile.flush()
# read CONNECT response
while self.client.rfile.readline() != b"\r\n":
pass
self.client.convert_to_ssl(alpn_protos=[b'h2'])
config = h2.config.H2Configuration(
client_side=True,
validate_outbound_headers=False,
validate_inbound_headers=False)
h2_conn = h2.connection.H2Connection(config)
h2_conn.initiate_connection()
self.client.wfile.write(h2_conn.data_to_send())
self.client.wfile.flush()
return h2_conn
def _setup_connection(self):
client = mitmproxy.net.tcp.TCPClient(("127.0.0.1", self.proxy.port))
client.connect()
# send CONNECT request
client.wfile.write(
http1.assemble_request(
mitmproxy.net.http.Request(
'authority',
b'CONNECT',
b'',
b'localhost',
self.server.server.address[1],
b'/',
b'HTTP/1.1',
[(b'host', b'localhost:%d' % self.server.server.address[1])
],
b'',
)))
client.wfile.flush()
# read CONNECT response
while client.rfile.readline() != b"\r\n":
pass
client.convert_to_ssl(alpn_protos=[b'h2'])
h2_conn = h2.connection.H2Connection(client_side=True,
header_encoding=False)
h2_conn.initiate_connection()
client.wfile.write(h2_conn.data_to_send())
client.wfile.flush()
return client, h2_conn
def patched_receive_handshake_data(
self, data
) -> layer.CommandGenerator[typing.Tuple[bool, typing.Optional[str]]]:
self.buf += data
response_head = self.buf.maybe_extract_lines()
if response_head:
response_head = [bytes(x) for x in response_head]
try:
response = http1.read_response_head(response_head)
except ValueError:
return True, None
challenge_message = extract_proxy_authenticate_msg(
response_head)
if 200 <= response.status_code < 300:
if self.buf:
yield from self.receive_data(data)
del self.buf
return True, None
else:
if not challenge_message:
return True, None
proxy_authorization = self.ntlm_context.get_ntlm_challenge_response_message(
challenge_message)
self.flow = build_connect_flow(
self.context,
("Proxy-Authorization", proxy_authorization))
raw = http1.assemble_request(self.flow.request)
yield commands.SendData(self.tunnel_connection, raw)
return False, None
else:
return False, None
def start_handshake(self) -> layer.CommandGenerator[None]:
if self.tunnel_connection.tls:
# "Secure Web Proxy": We may have negotiated an ALPN when connecting to the upstream proxy.
# The semantics are not really clear here, but we make sure that if we negotiated h2,
# we act as an h2 client.
self.conn.alpn = self.tunnel_connection.alpn
if not self.send_connect:
return (yield from super().start_handshake())
assert self.conn.address
req = http.Request(
host=self.conn.address[0],
port=self.conn.address[1],
method=b"CONNECT",
scheme=b"",
authority=f"{self.conn.address[0]}:{self.conn.address[1]}".encode(
),
path=b"",
http_version=b"HTTP/1.1",
headers=http.Headers(),
content=b"",
trailers=None,
timestamp_start=time.time(),
timestamp_end=time.time(),
)
raw = http1.assemble_request(req)
yield commands.SendData(self.tunnel_connection, raw)
def _setup_connection(self):
client = mitmproxy.net.tcp.TCPClient(("127.0.0.1", self.proxy.port))
client.connect()
# send CONNECT request
client.wfile.write(http1.assemble_request(mitmproxy.net.http.Request(
'authority',
b'CONNECT',
b'',
b'localhost',
self.server.server.address.port,
b'/',
b'HTTP/1.1',
[(b'host', b'localhost:%d' % self.server.server.address.port)],
b'',
)))
client.wfile.flush()
# read CONNECT response
while client.rfile.readline() != b"\r\n":
pass
client.convert_to_ssl(alpn_protos=[b'h2'])
h2_conn = h2.connection.H2Connection(client_side=True, header_encoding=False)
h2_conn.initiate_connection()
client.wfile.write(h2_conn.data_to_send())
client.wfile.flush()
return client, h2_conn
def setup_connection(self):
self.client = mitmproxy.net.tcp.TCPClient(
("127.0.0.1", self.proxy.port))
self.client.connect()
# send CONNECT request
self.client.wfile.write(
http1.assemble_request(
mitmproxy.http.make_connect_request(
("localhost", self.server.server.address[1]))))
self.client.wfile.flush()
# read CONNECT response
while self.client.rfile.readline() != b"\r\n":
pass
self.client.convert_to_tls(alpn_protos=[b'h2'])
config = h2.config.H2Configuration(client_side=True,
validate_outbound_headers=False,
validate_inbound_headers=False)
h2_conn = h2.connection.H2Connection(config)
h2_conn.initiate_connection()
self.client.wfile.write(h2_conn.data_to_send())
self.client.wfile.flush()
return h2_conn
def patched_start_handshake(self) -> layer.CommandGenerator[None]:
assert self.conn.address
self.ntlm_context = CustomNTLMContext(ctx)
proxy_authorization = self.ntlm_context.get_ntlm_start_negotiate_message(
)
self.flow = build_connect_flow(
self.context, ("Proxy-Authorization", proxy_authorization))
yield HttpConnectUpstreamHook(self.flow)
raw = http1.assemble_request(self.flow.request)
yield commands.SendData(self.tunnel_connection, raw)
def start_handshake(self) -> layer.CommandGenerator[None]:
if self.tunnel_connection.tls:
# "Secure Web Proxy": We may have negotiated an ALPN when connecting to the upstream proxy.
# The semantics are not really clear here, but we make sure that if we negotiated h2,
# we act as an h2 client.
self.conn.alpn = self.tunnel_connection.alpn
if not self.send_connect:
return (yield from super().start_handshake())
assert self.conn.address
req = http.make_connect_request(self.conn.address)
raw = http1.assemble_request(req)
yield commands.SendData(self.tunnel_connection, raw)
def __call__(self):
self.connect()
client = self.client_conn.connection
server = self.server_conn.connection
buf = memoryview(bytearray(self.chunk_size))
# send CONNECT, expect 200 OK
connect_req = make_connect_request((self.host, self.port))
server.send(assemble_request(connect_req))
resp = server.recv(1024).decode()
if not resp.startswith('HTTP/1.1 200 OK'):
raise BubbleFlexPassthruException('CONNECT request error: ' + resp)
conns = [client, server]
# https://github.com/openssl/openssl/issues/6234
for conn in conns:
if isinstance(conn, SSL.Connection) and hasattr(
SSL._lib, "SSL_clear_mode"):
SSL._lib.SSL_clear_mode(conn._ssl,
SSL._lib.SSL_MODE_AUTO_RETRY)
try:
while not self.channel.should_exit.is_set():
r = ssl_read_select(conns, 10)
for conn in r:
dst = server if conn == client else client
try:
size = conn.recv_into(buf, self.chunk_size)
except (SSL.WantReadError, SSL.WantWriteError):
continue
if not size:
conns.remove(conn)
# Shutdown connection to the other peer
if isinstance(conn, SSL.Connection):
# We can't half-close a connection, so we just close everything here.
# Sockets will be cleaned up on a higher level.
return
else:
dst.shutdown(socket.SHUT_WR)
if len(conns) == 0:
return
continue
tcp_message = tcp.TCPMessage(dst == server,
buf[:size].tobytes())
dst.sendall(tcp_message.content)
except (socket.error, exceptions.TcpException, SSL.Error) as e:
bubble_log.error('exception: ' + repr(e))
def test_simple(self):
self.d = test.Daemon()
sc = connections.ServerConnection((self.d.IFACE, self.d.port))
sc.connect()
f = tflow.tflow()
f.server_conn = sc
f.request.path = "/p/200:da"
# use this protocol just to assemble - not for actual sending
sc.wfile.write(http1.assemble_request(f.request))
sc.wfile.flush()
assert http1.read_response(sc.rfile, f.request, 1000)
assert self.d.last_log()
sc.finish()
self.d.shutdown()
def test_simple(self):
d = test.Daemon()
c = connections.ServerConnection((d.IFACE, d.port))
c.connect()
f = tflow.tflow()
f.server_conn = c
f.request.path = "/p/200:da"
# use this protocol just to assemble - not for actual sending
c.wfile.write(http1.assemble_request(f.request))
c.wfile.flush()
assert http1.read_response(c.rfile, f.request, 1000)
assert d.last_log()
c.finish()
d.shutdown()
def start_handshake(self) -> layer.CommandGenerator[None]:
if not self.send_connect:
return (yield from super().start_handshake())
assert self.conn.address
flow = http.HTTPFlow(self.context.client, self.tunnel_connection)
flow.request = http.Request(
host=self.conn.address[0],
port=self.conn.address[1],
method=b"CONNECT",
scheme=b"",
authority=f"{self.conn.address[0]}:{self.conn.address[1]}".encode(
),
path=b"",
http_version=b"HTTP/1.1",
headers=http.Headers(),
content=b"",
trailers=None,
timestamp_start=time.time(),
timestamp_end=time.time(),
)
yield HttpConnectUpstreamHook(flow)
raw = http1.assemble_request(flow.request)
yield commands.SendData(self.tunnel_connection, raw)
def send_request(self, request):
self.server_conn.wfile.write(http1.assemble_request(request))
self.server_conn.wfile.flush()
def run(self):
r = self.f.request
first_line_format_backup = r.first_line_format
server = None
try:
self.f.response = None
# If we have a channel, run script hooks.
if self.channel:
request_reply = self.channel.ask("request", self.f)
if isinstance(request_reply, http.HTTPResponse):
self.f.response = request_reply
if not self.f.response:
# In all modes, we directly connect to the server displayed
if self.config.options.mode == "upstream":
server_address = self.config.upstream_server.address
server = connections.ServerConnection(server_address, (self.config.options.listen_host, 0))
server.connect()
if r.scheme == "https":
connect_request = http.make_connect_request((r.data.host, r.port))
server.wfile.write(http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(
server.rfile,
connect_request,
body_size_limit=self.config.options.body_size_limit
)
if resp.status_code != 200:
raise exceptions.ReplayException("Upstream server refuses CONNECT request")
server.establish_ssl(
self.config.clientcerts,
sni=self.f.server_conn.sni
)
r.first_line_format = "relative"
else:
r.first_line_format = "absolute"
else:
server_address = (r.host, r.port)
server = connections.ServerConnection(
server_address,
(self.config.options.listen_host, 0)
)
server.connect()
if r.scheme == "https":
server.establish_ssl(
self.config.clientcerts,
sni=self.f.server_conn.sni
)
r.first_line_format = "relative"
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
self.f.server_conn = server
self.f.response = http.HTTPResponse.wrap(
http1.read_response(
server.rfile,
r,
body_size_limit=self.config.options.body_size_limit
)
)
if self.channel:
response_reply = self.channel.ask("response", self.f)
if response_reply == exceptions.Kill:
raise exceptions.Kill()
except (exceptions.ReplayException, exceptions.NetlibException) as e:
self.f.error = flow.Error(str(e))
if self.channel:
self.channel.ask("error", self.f)
except exceptions.Kill:
# Kill should only be raised if there's a channel in the
# first place.
self.channel.tell(
"log",
log.LogEntry("Connection killed", "info")
)
except Exception:
self.channel.tell(
"log",
log.LogEntry(traceback.format_exc(), "error")
)
finally:
r.first_line_format = first_line_format_backup
self.f.live = False
if server.connected():
server.finish()
def replay(self, f): # pragma: no cover
f.live = True
r = f.request
bsl = human.parse_size(self.options.body_size_limit)
first_line_format_backup = r.first_line_format
server = None
global new, cur_cycle, cur_group
try:
f.response = None
# If we have a channel, run script hooks.
request_reply = self.channel.ask("request", f)
if isinstance(request_reply, http.HTTPResponse):
f.response = request_reply
if not f.response:
# In all modes, we directly connect to the server displayed
if self.options.mode.startswith("upstream:"):
server_address = server_spec.parse_with_mode(
self.options.mode)[1].address
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
connect_request = http.make_connect_request(
(r.data.host, r.port))
server.wfile.write(
http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(server.rfile,
connect_request,
body_size_limit=bsl)
if resp.status_code != 200:
raise exceptions.ReplayException(
"Upstream server refuses CONNECT request")
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.first_line_format = "relative"
else:
r.first_line_format = "absolute"
else:
server_address = (r.host, r.port)
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.first_line_format = "relative"
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
if f.server_conn:
f.server_conn.close()
f.server_conn = server
f.response = http.HTTPResponse.wrap(
http1.read_response(server.rfile, r, body_size_limit=bsl))
response_reply = self.channel.ask("response", f)
#new.append(f) #record the response
cur_cycle[cur_group] = f
if response_reply == exceptions.Kill:
raise exceptions.Kill()
except (exceptions.ReplayException, exceptions.NetlibException) as e:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
except exceptions.Kill:
self.channel.tell("log", log.LogEntry("Connection killed", "info"))
except Exception as e:
self.channel.tell("log", log.LogEntry(repr(e), "error"))
finally:
r.first_line_format = first_line_format_backup
f.live = False
if server.connected():
server.finish()
server.close()
def replay(self, f): # pragma: no cover
f.live = True
r = f.request
bsl = human.parse_size(self.options.body_size_limit)
first_line_format_backup = r.first_line_format
server = None
try:
f.response = None
# If we have a channel, run script hooks.
request_reply = self.channel.ask("request", f)
if isinstance(request_reply, http.HTTPResponse):
f.response = request_reply
if not f.response:
# In all modes, we directly connect to the server displayed
if self.options.mode.startswith("upstream:"):
server_address = server_spec.parse_with_mode(self.options.mode)[1].address
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
connect_request = http.make_connect_request((r.data.host, r.port))
server.wfile.write(http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(
server.rfile,
connect_request,
body_size_limit=bsl
)
if resp.status_code != 200:
raise exceptions.ReplayException(
"Upstream server refuses CONNECT request"
)
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options)
)
r.first_line_format = "relative"
else:
r.first_line_format = "absolute"
else:
server_address = (r.host, r.port)
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options)
)
r.first_line_format = "relative"
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
r.timestamp_start = r.timestamp_end = time.time()
if f.server_conn:
f.server_conn.close()
f.server_conn = server
f.response = http.HTTPResponse.wrap(
http1.read_response(server.rfile, r, body_size_limit=bsl)
)
response_reply = self.channel.ask("response", f)
if response_reply == exceptions.Kill:
raise exceptions.Kill()
except (exceptions.ReplayException, exceptions.NetlibException) as e:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
except exceptions.Kill:
self.channel.tell("log", log.LogEntry("Connection killed", "info"))
except Exception as e:
self.channel.tell("log", log.LogEntry(repr(e), "error"))
finally:
r.first_line_format = first_line_format_backup
f.live = False
if server.connected():
server.finish()
server.close()
def send_request(self, request):
# TODO: this does not yet support request trailers
self.server_conn.wfile.write(http1.assemble_request(request))
self.server_conn.wfile.flush()
def send_request(self, request):
self.server_conn.wfile.write(http1.assemble_request(request))
self.server_conn.wfile.flush()
def replay(self, f): # pragma: no cover
f.live = True
r = f.request
bsl = human.parse_size(self.options.body_size_limit)
authority_backup = r.authority
server = None
try:
f.response = None
# If we have a channel, run script hooks.
request_reply = self.channel.ask("request", f)
if isinstance(request_reply, http.HTTPResponse):
f.response = request_reply
if not f.response:
# In all modes, we directly connect to the server displayed
if self.options.mode.startswith("upstream:"):
server_address = server_spec.parse_with_mode(
self.options.mode)[1].address
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
connect_request = http.make_connect_request(
(r.data.host, r.port))
server.wfile.write(
http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(server.rfile,
connect_request,
body_size_limit=bsl)
if resp.status_code != 200:
raise exceptions.ReplayException(
"Upstream server refuses CONNECT request")
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.authority = b""
else:
r.authority = hostport(r.scheme, r.host, r.port)
else:
server_address = (r.host, r.port)
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.authority = ""
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
r.timestamp_start = r.timestamp_end = time.time()
if f.server_conn:
f.server_conn.close()
f.server_conn = server
f.response = http1.read_response(server.rfile,
r,
body_size_limit=bsl)
response_reply = self.channel.ask("response", f)
if response_reply == exceptions.Kill:
raise exceptions.Kill()
except (exceptions.ReplayException, exceptions.NetlibException) as e:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
except exceptions.Kill:
self.channel.tell("log",
log.LogEntry(flow.Error.KILLED_MESSAGE, "info"))
except Exception as e:
self.channel.tell("log", log.LogEntry(repr(e), "error"))
finally:
r.authority = authority_backup
f.live = False
if server and server.connected():
server.finish()
server.close()
