def ask(self, mtype, m):
"""
Decorate a message with a reply attribute, and send it to the
master. Then wait for a response.
Raises:
exceptions.Kill: All connections should be closed immediately.
"""
m.reply = Reply(m)
self.q.put((mtype, m))
while not self.should_exit.is_set():
try:
# The timeout is here so we can handle a should_exit event.
g = m.reply.q.get(timeout=0.5)
except queue.Empty: # pragma: no cover
continue
if g == exceptions.Kill:
raise exceptions.Kill()
return g
raise exceptions.Kill()
def ask(self, mtype, m):
"""
Decorate a message with a reply attribute, and send it to the master.
Then wait for a response.
Raises:
exceptions.Kill: All connections should be closed immediately.
"""
m.reply = Reply(m)
asyncio.run_coroutine_threadsafe(self._q.put((mtype, m)), self.loop)
g = m.reply.q.get()
if g == exceptions.Kill:
raise exceptions.Kill()
return g
def ask(self, mtype, m):
"""
Decorate a message with a reply attribute, and send it to the master.
Then wait for a response.
Raises:
exceptions.Kill: All connections should be closed immediately.
"""
if not self.should_exit.is_set():
m.reply = Reply(m)
asyncio.run_coroutine_threadsafe(
self.master.addons.handle_lifecycle(mtype, m),
self.loop,
)
g = m.reply.q.get()
if g == exceptions.Kill:
raise exceptions.Kill()
return g
def run(self):
r = self.f.request
first_line_format_backup = r.first_line_format
server = None
try:
self.f.response = None
# If we have a channel, run script hooks.
if self.channel:
request_reply = self.channel.ask("request", self.f)
if isinstance(request_reply, http.HTTPResponse):
self.f.response = request_reply
if not self.f.response:
# In all modes, we directly connect to the server displayed
if self.config.options.mode == "upstream":
server_address = self.config.upstream_server.address
server = connections.ServerConnection(server_address, (self.config.options.listen_host, 0))
server.connect()
if r.scheme == "https":
connect_request = http.make_connect_request((r.data.host, r.port))
server.wfile.write(http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(
server.rfile,
connect_request,
body_size_limit=self.config.options.body_size_limit
)
if resp.status_code != 200:
raise exceptions.ReplayException("Upstream server refuses CONNECT request")
server.establish_ssl(
self.config.clientcerts,
sni=self.f.server_conn.sni
)
r.first_line_format = "relative"
else:
r.first_line_format = "absolute"
else:
server_address = (r.host, r.port)
server = connections.ServerConnection(
server_address,
(self.config.options.listen_host, 0)
)
server.connect()
if r.scheme == "https":
server.establish_ssl(
self.config.clientcerts,
sni=self.f.server_conn.sni
)
r.first_line_format = "relative"
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
self.f.server_conn = server
self.f.response = http.HTTPResponse.wrap(
http1.read_response(
server.rfile,
r,
body_size_limit=self.config.options.body_size_limit
)
)
if self.channel:
response_reply = self.channel.ask("response", self.f)
if response_reply == exceptions.Kill:
raise exceptions.Kill()
except (exceptions.ReplayException, exceptions.NetlibException) as e:
self.f.error = flow.Error(str(e))
if self.channel:
self.channel.ask("error", self.f)
except exceptions.Kill:
# Kill should only be raised if there's a channel in the
# first place.
self.channel.tell(
"log",
log.LogEntry("Connection killed", "info")
)
except Exception:
self.channel.tell(
"log",
log.LogEntry(traceback.format_exc(), "error")
)
finally:
r.first_line_format = first_line_format_backup
self.f.live = False
if server.connected():
server.finish()
def replay(self, f): # pragma: no cover
f.live = True
r = f.request
bsl = human.parse_size(self.options.body_size_limit)
first_line_format_backup = r.first_line_format
server = None
global new, cur_cycle, cur_group
try:
f.response = None
# If we have a channel, run script hooks.
request_reply = self.channel.ask("request", f)
if isinstance(request_reply, http.HTTPResponse):
f.response = request_reply
if not f.response:
# In all modes, we directly connect to the server displayed
if self.options.mode.startswith("upstream:"):
server_address = server_spec.parse_with_mode(
self.options.mode)[1].address
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
connect_request = http.make_connect_request(
(r.data.host, r.port))
server.wfile.write(
http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(server.rfile,
connect_request,
body_size_limit=bsl)
if resp.status_code != 200:
raise exceptions.ReplayException(
"Upstream server refuses CONNECT request")
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.first_line_format = "relative"
else:
r.first_line_format = "absolute"
else:
server_address = (r.host, r.port)
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.first_line_format = "relative"
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
if f.server_conn:
f.server_conn.close()
f.server_conn = server
f.response = http.HTTPResponse.wrap(
http1.read_response(server.rfile, r, body_size_limit=bsl))
response_reply = self.channel.ask("response", f)
#new.append(f) #record the response
cur_cycle[cur_group] = f
if response_reply == exceptions.Kill:
raise exceptions.Kill()
except (exceptions.ReplayException, exceptions.NetlibException) as e:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
except exceptions.Kill:
self.channel.tell("log", log.LogEntry("Connection killed", "info"))
except Exception as e:
self.channel.tell("log", log.LogEntry(repr(e), "error"))
finally:
r.first_line_format = first_line_format_backup
f.live = False
if server.connected():
server.finish()
server.close()
def replay(self, f): # pragma: no cover
f.live = True
r = f.request
bsl = human.parse_size(self.options.body_size_limit)
authority_backup = r.authority
server = None
try:
f.response = None
# If we have a channel, run script hooks.
request_reply = self.channel.ask("request", f)
if isinstance(request_reply, http.HTTPResponse):
f.response = request_reply
if not f.response:
# In all modes, we directly connect to the server displayed
if self.options.mode.startswith("upstream:"):
server_address = server_spec.parse_with_mode(
self.options.mode)[1].address
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
connect_request = http.make_connect_request(
(r.data.host, r.port))
server.wfile.write(
http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(server.rfile,
connect_request,
body_size_limit=bsl)
if resp.status_code != 200:
raise exceptions.ReplayException(
"Upstream server refuses CONNECT request")
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.authority = b""
else:
r.authority = hostport(r.scheme, r.host, r.port)
else:
server_address = (r.host, r.port)
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.authority = ""
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
r.timestamp_start = r.timestamp_end = time.time()
if f.server_conn:
f.server_conn.close()
f.server_conn = server
f.response = http1.read_response(server.rfile,
r,
body_size_limit=bsl)
response_reply = self.channel.ask("response", f)
if response_reply == exceptions.Kill:
raise exceptions.Kill()
except (exceptions.ReplayException, exceptions.NetlibException) as e:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
except exceptions.Kill:
self.channel.tell("log",
log.LogEntry(flow.Error.KILLED_MESSAGE, "info"))
except Exception as e:
self.channel.tell("log", log.LogEntry(repr(e), "error"))
finally:
r.authority = authority_backup
f.live = False
if server and server.connected():
server.finish()
server.close()
def _next_layer(self, top_layer):
logging.info('root_context._next_layer(%s)', top_layer)
try:
d = top_layer.client_conn.rfile.peek(3)
except exceptions.TcpException as e:
logging.info('ignoring protocol: %s', e)
raise exceptions.ProtocolException(str(e))
client_tls = tls.is_tls_record_magic(d)
host, port = '(unknown)', '(unknown)'
# 1. check for filter
if self.config.check_filter:
if top_layer.server_conn.address:
host, port = top_layer.server_conn.address
is_filtered = self.config.check_filter(
top_layer.server_conn.address)
logging.info('_next_layer: %s is_filtered: %s',
vars(top_layer.server_conn), is_filtered)
if not is_filtered and client_tls:
try:
client_hello = tls.ClientHello.from_file(
self.client_conn.rfile)
except exceptions.TlsProtocolException as e:
self.log("Cannot parse Client Hello: %s" % repr(e),
"error")
else:
sni_str = client_hello.sni and client_hello.sni.decode(
"idna")
host, port = (sni_str, 443)
is_filtered = self.config.check_filter((host, port))
logging.info('is_filtered: %s', is_filtered)
if is_filtered:
if self.config.options.block_not_ignore:
self.log(
'-> {}:{} {} blocked'.format(host, port,
datetime.now()), 'info')
raise exceptions.Kill(
f'blocked https request to filtered host {host}')
else:
return protocol.RawTCPLayer(top_layer, ignore=True)
else:
self.log(
'-> {}:{} {} allowed'.format(host, port, datetime.now()),
'info')
# 2. Always insert a TLS layer, even if there's neither client nor server tls.
# An inline script may upgrade from http to https,
# in which case we need some form of TLS layer.
if isinstance(top_layer, modes.ReverseProxy):
return protocol.TlsLayer(top_layer, client_tls,
top_layer.server_tls,
top_layer.server_conn.address[0])
if isinstance(top_layer, protocol.ServerConnectionMixin):
return protocol.TlsLayer(top_layer, client_tls, client_tls)
if isinstance(top_layer, protocol.UpstreamConnectLayer):
# if the user manually sets a scheme for connect requests, we use this to decide if we
# want TLS or not.
if top_layer.connect_request.scheme:
server_tls = top_layer.connect_request.scheme == "https"
else:
server_tls = client_tls
return protocol.TlsLayer(top_layer, client_tls, server_tls)
# 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed.
if isinstance(top_layer, protocol.TlsLayer):
if isinstance(top_layer.ctx, modes.HttpProxy):
return protocol.Http1Layer(top_layer, http.HTTPMode.regular)
if isinstance(top_layer.ctx, modes.HttpUpstreamProxy):
return protocol.Http1Layer(top_layer, http.HTTPMode.upstream)
# 4. Check for other TLS cases (e.g. after CONNECT).
if client_tls:
return protocol.TlsLayer(top_layer, True, True)
# 4. Check for --tcp
if self.config.check_tcp(top_layer.server_conn.address):
return protocol.RawTCPLayer(top_layer)
# 5. Check for TLS ALPN (HTTP1/HTTP2)
if isinstance(top_layer, protocol.TlsLayer):
alpn = top_layer.client_conn.get_alpn_proto_negotiated()
if alpn == b'h2':
return protocol.Http2Layer(top_layer,
http.HTTPMode.transparent)
if alpn == b'http/1.1':
return protocol.Http1Layer(top_layer,
http.HTTPMode.transparent)
# 6. Check for raw tcp mode
is_ascii = (
len(d) == 3 and
# expect A-Za-z
all(65 <= x <= 90 or 97 <= x <= 122 for x in d))
if self.config.options.rawtcp and not is_ascii:
return protocol.RawTCPLayer(top_layer)
# 7. Assume HTTP1 by default
return protocol.Http1Layer(top_layer, http.HTTPMode.transparent)
