def handle_flow(self):
flow = HTTPFlow(self.c.client_conn, self.c.server_conn,
self.change_server)
try:
req = HTTPRequest.from_stream(
self.c.client_conn.rfile,
body_size_limit=self.c.config.body_size_limit)
self.c.log("request", "debug",
[req._assemble_first_line(req.form_in)])
send_upstream = self.process_request(flow, req)
if not send_upstream:
return True
# Be careful NOT to assign the request to the flow before
# process_request completes. This is because the call can raise an
# exception. If the request object is already attached, this results
# in an Error object that has an attached request that has not been
# sent through to the Master.
flow.request = req
request_reply = self.c.channel.ask("request", flow.request)
flow.server_conn = self.c.server_conn
if request_reply is None or request_reply == KILL:
return False
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
else:
flow.response = self.get_response_from_server(flow.request)
flow.server_conn = self.c.server_conn # no further manipulation of self.c.server_conn beyond this point
# we can safely set it as the final attribute value here.
self.c.log("response", "debug",
[flow.response._assemble_first_line()])
response_reply = self.c.channel.ask("response", flow.response)
if response_reply is None or response_reply == KILL:
return False
self.c.client_conn.send(flow.response._assemble())
flow.timestamp_end = utils.timestamp()
if (http.connection_close(flow.request.httpversion,
flow.request.headers)
or http.connection_close(flow.response.httpversion,
flow.response.headers)):
return False
if flow.request.form_in == "authority":
self.ssl_upgrade()
# If the user has changed the target server on this connection,
# restore the original target server
self.restore_server()
return True
except (HttpAuthenticationError, http.HttpError, proxy.ProxyError,
tcp.NetLibError), e:
self.handle_error(e, flow)
def handle_flow(self):
flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.change_server)
try:
req = HTTPRequest.from_stream(self.c.client_conn.rfile,
body_size_limit=self.c.config.body_size_limit)
self.c.log("request", "debug", [req._assemble_first_line(req.form_in)])
send_request_upstream = self.process_request(flow, req)
if not send_request_upstream:
return True
# Be careful NOT to assign the request to the flow before
# process_request completes. This is because the call can raise an
# exception. If the request object is already attached, this results
# in an Error object that has an attached request that has not been
# sent through to the Master.
flow.request = req
request_reply = self.c.channel.ask("request", flow.request)
flow.server_conn = self.c.server_conn
if request_reply is None or request_reply == KILL:
return False
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
else:
flow.response = self.get_response_from_server(flow.request)
flow.server_conn = self.c.server_conn # no further manipulation of self.c.server_conn beyond this point
# we can safely set it as the final attribute value here.
self.c.log("response", "debug", [flow.response._assemble_first_line()])
response_reply = self.c.channel.ask("response", flow.response)
if response_reply is None or response_reply == KILL:
return False
self.c.client_conn.send(flow.response._assemble())
flow.timestamp_end = utils.timestamp()
if (http.connection_close(flow.request.httpversion, flow.request.headers) or
http.connection_close(flow.response.httpversion, flow.response.headers)):
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return False
if flow.request.form_in == "authority" and flow.response.code == 200:
self.ssl_upgrade()
# If the user has changed the target server on this connection,
# restore the original target server
self.restore_server()
return True
except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e:
self.handle_error(e, flow)
def test_connection_close():
h = odict.ODictCaseless()
assert http.connection_close((1, 0), h)
assert not http.connection_close((1, 1), h)
h["connection"] = ["keep-alive"]
assert not http.connection_close((1, 1), h)
h["connection"] = ["close"]
assert http.connection_close((1, 1), h)
def test_connection_close():
h = odict.ODictCaseless()
assert http.connection_close((1, 0), h)
assert not http.connection_close((1, 1), h)
h["connection"] = ["keep-alive"]
assert not http.connection_close((1, 1), h)
h["connection"] = ["close"]
assert http.connection_close((1, 1), h)
def check_close_connection(self, flow):
"""
Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so.
"""
close_connection = (
http.connection_close(flow.request.httpversion, flow.request.headers) or
http.connection_close(flow.response.httpversion, flow.response.headers) or
http.expected_http_body_size(flow.response.headers, False, flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return True
return False
def check_close_connection(self, flow):
"""
Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so.
"""
close_connection = (http.connection_close(flow.request.httpversion,
flow.request.headers)
or http.connection_close(flow.response.httpversion,
flow.response.headers)
or http.expected_http_body_size(
flow.response.headers, False,
flow.request.method, flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return True
return False
def handle_flow(self):
flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live)
try:
req = HTTPRequest.from_stream(self.c.client_conn.rfile,
body_size_limit=self.c.config.body_size_limit)
self.c.log("request", "debug", [req._assemble_first_line(req.form_in)])
send_request_upstream = self.process_request(flow, req)
if not send_request_upstream:
return True
# Be careful NOT to assign the request to the flow before
# process_request completes. This is because the call can raise an
# exception. If the request object is already attached, this results
# in an Error object that has an attached request that has not been
# sent through to the Master.
flow.request = req
request_reply = self.c.channel.ask("request", flow.request)
self.determine_server_address(flow, flow.request)
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow
if request_reply is None or request_reply == KILL:
return False
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
else:
# read initially in "stream" mode, so we can get the headers separately
flow.response = self.get_response_from_server(flow.request, include_body=False)
# call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True
self.c.channel.ask("responseheaders", flow.response)
# now get the rest of the request body, if body still needs to be read but not streaming this response
if flow.response.stream:
flow.response.content = CONTENT_MISSING
else:
flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit,
flow.request.method, flow.response.code, False)
# no further manipulation of self.c.server_conn beyond this point
# we can safely set it as the final attribute value here.
flow.server_conn = self.c.server_conn
self.c.log("response", "debug", [flow.response._assemble_first_line()])
response_reply = self.c.channel.ask("response", flow.response)
if response_reply is None or response_reply == KILL:
return False
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can send it to the client straight away.
self.c.client_conn.send(flow.response._assemble())
else:
# streaming:
# First send the body and then transfer the response incrementally:
h = flow.response._assemble_head(preserve_transfer_encoding=True)
self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False, 4096):
for part in chunk:
self.c.client_conn.wfile.write(part)
self.c.client_conn.wfile.flush()
flow.response.timestamp_end = utils.timestamp()
flow.timestamp_end = utils.timestamp()
close_connection = (
http.connection_close(flow.request.httpversion, flow.request.headers) or
http.connection_close(flow.response.httpversion, flow.response.headers) or
http.expected_http_body_size(flow.response.headers, False, flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return False
if flow.request.form_in == "authority" and flow.response.code == 200:
# TODO: Eventually add headers (space/usefulness tradeoff)
# Make sure to add state info before the actual upgrade happens.
# During the upgrade, we may receive an SNI indication from the client,
# which resets the upstream connection. If this is the case, we must
# already re-issue the CONNECT request at this point.
self.c.server_conn.state.append(("http", {"state": "connect",
"host": flow.request.host,
"port": flow.request.port}))
self.ssl_upgrade()
# If the user has changed the target server on this connection,
# restore the original target server
flow.live.restore_server()
flow.live = None
return True
except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e:
self.handle_error(e, flow)
def handle_request(self, cc):
try:
request, err = None, None
request = self.read_request(cc)
if request is None:
return
cc.requestcount += 1
request_reply = self.channel.ask("request", request)
if request_reply is None or request_reply == KILL:
return
elif isinstance(request_reply, flow.Response):
request = False
response = request_reply
response_reply = self.channel.ask("response", response)
else:
request = request_reply
if self.config.reverse_proxy:
scheme, host, port = self.config.reverse_proxy
elif self.config.forward_proxy:
scheme, host, port = self.config.forward_proxy
else:
scheme, host, port = request.scheme, request.host, request.port
# If we've already pumped a request over this connection,
# it's possible that the server has timed out. If this is
# the case, we want to reconnect without sending an error
# to the client.
while 1:
sc = self.get_server_connection(cc, scheme, host, port, self.sni, request=request)
sc.send(request)
if sc.requestcount == 1: # add timestamps only for first request (others are not directly affected)
request.tcp_setup_timestamp = sc.tcp_setup_timestamp
request.ssl_setup_timestamp = sc.ssl_setup_timestamp
sc.rfile.reset_timestamps()
try:
peername = sc.connection.getpeername()
if peername:
request.ip = peername[0]
httpversion, code, msg, headers, content = http.read_response(
sc.rfile,
request.method,
self.config.body_size_limit
)
except http.HttpErrorConnClosed:
self.del_server_connection()
if sc.requestcount > 1:
continue
else:
raise
except http.HttpError:
raise ProxyError(502, "Invalid server response.")
else:
break
response = flow.Response(
request, httpversion, code, msg, headers, content, sc.cert,
sc.rfile.first_byte_timestamp
)
response_reply = self.channel.ask("response", response)
# Not replying to the server invalidates the server
# connection, so we terminate.
if response_reply == KILL:
sc.terminate()
if response_reply == KILL:
return
else:
response = response_reply
self.send_response(response)
if request and http.connection_close(request.httpversion, request.headers):
return
# We could keep the client connection when the server
# connection needs to go away. However, we want to mimic
# behaviour as closely as possible to the client, so we
# disconnect.
if http.connection_close(response.httpversion, response.headers):
return
except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e:
if hasattr(e, "code"):
cc.error = "%s: %s"%(e.code, e.msg)
else:
cc.error = str(e)
if request:
err = flow.Error(request, cc.error)
self.channel.ask("error", err)
self.log(
cc, cc.error,
["url: %s"%request.get_url()]
)
else:
self.log(cc, cc.error)
if isinstance(e, ProxyError):
self.send_error(e.code, e.msg, e.headers)
response = flow.Response(request, httpversion, code, msg,
headers, content, sc.cert,
sc.rfile.first_byte_timestamp)
response_reply = self.channel.ask(response)
# Not replying to the server invalidates the server
# connection, so we terminate.
if response_reply == KILL:
sc.terminate()
if response_reply == KILL:
return
else:
response = response_reply
self.send_response(response)
if request and http.connection_close(
request.httpversion, request.headers):
return
# We could keep the client connection when the server
# connection needs to go away. However, we want to mimic
# behaviour as closely as possible to the client, so we
# disconnect.
if http.connection_close(response.httpversion,
response.headers):
return
except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e:
if hasattr(e, "code"):
cc.error = "%s: %s" % (e.code, e.msg)
else:
cc.error = str(e)
if request:
response = flow.Response(
request, httpversion, code, msg, headers, content, sc.cert,
sc.rfile.first_byte_timestamp
)
response_reply = self.channel.ask(response)
# Not replying to the server invalidates the server
# connection, so we terminate.
if response_reply == KILL:
sc.terminate()
if response_reply == KILL:
return
else:
response = response_reply
self.send_response(response)
if request and http.connection_close(request.httpversion, request.headers):
return
# We could keep the client connection when the server
# connection needs to go away. However, we want to mimic
# behaviour as closely as possible to the client, so we
# disconnect.
if http.connection_close(response.httpversion, response.headers):
return
except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e:
if hasattr(e, "code"):
cc.error = "%s: %s"%(e.code, e.msg)
else:
cc.error = str(e)
if request:
err = flow.Error(request, cc.error)
def handle_flow(self):
flow = HTTPFlow(self.c.client_conn, self.c.server_conn,
self.change_server)
try:
req = HTTPRequest.from_stream(
self.c.client_conn.rfile,
body_size_limit=self.c.config.body_size_limit)
self.c.log("request", "debug",
[req._assemble_first_line(req.form_in)])
send_request_upstream = self.process_request(flow, req)
if not send_request_upstream:
return True
# Be careful NOT to assign the request to the flow before
# process_request completes. This is because the call can raise an
# exception. If the request object is already attached, this results
# in an Error object that has an attached request that has not been
# sent through to the Master.
flow.request = req
request_reply = self.c.channel.ask("request", flow.request)
self.determine_server_address(flow, flow.request)
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow
if request_reply is None or request_reply == KILL:
return False
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
else:
# read initially in "stream" mode, so we can get the headers separately
flow.response = self.get_response_from_server(
flow.request, include_body=False)
# call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True
self.c.channel.ask("responseheaders", flow.response)
# now get the rest of the request body, if body still needs to be read but not streaming this response
if flow.response.stream:
flow.response.content = CONTENT_MISSING
else:
flow.response.content = http.read_http_body(
self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False)
# no further manipulation of self.c.server_conn beyond this point
# we can safely set it as the final attribute value here.
flow.server_conn = self.c.server_conn
self.c.log("response", "debug",
[flow.response._assemble_first_line()])
response_reply = self.c.channel.ask("response", flow.response)
if response_reply is None or response_reply == KILL:
return False
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can send it to the client straight away.
self.c.client_conn.send(flow.response._assemble())
else:
# streaming:
# First send the body and then transfer the response incrementally:
h = flow.response._assemble_head(
preserve_transfer_encoding=True)
self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(
self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False, 4096):
for part in chunk:
self.c.client_conn.wfile.write(part)
self.c.client_conn.wfile.flush()
flow.timestamp_end = utils.timestamp()
close_connection = (
http.connection_close(flow.request.httpversion,
flow.request.headers)
or http.connection_close(flow.response.httpversion,
flow.response.headers)
or http.expected_http_body_size(flow.response.headers, False,
flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return False
if flow.request.form_in == "authority" and flow.response.code == 200:
# TODO: Eventually add headers (space/usefulness tradeoff)
# Make sure to add state info before the actual upgrade happens.
# During the upgrade, we may receive an SNI indication from the client,
# which resets the upstream connection. If this is the case, we must
# already re-issue the CONNECT request at this point.
self.c.server_conn.state.append(("http", {
"state": "connect",
"host": flow.request.host,
"port": flow.request.port
}))
self.ssl_upgrade()
# If the user has changed the target server on this connection,
# restore the original target server
self.restore_server()
return True
except (HttpAuthenticationError, http.HttpError, proxy.ProxyError,
tcp.NetLibError), e:
self.handle_error(e, flow)