def test_expected_http_body_size():
# gibber in the content-length field
h = odict.ODictCaseless()
h["content-length"] = ["foo"]
assert http.expected_http_body_size(h, False, "GET", 200) is None
# negative number in the content-length field
h = odict.ODictCaseless()
h["content-length"] = ["-7"]
assert http.expected_http_body_size(h, False, "GET", 200) is None
# explicit length
h = odict.ODictCaseless()
h["content-length"] = ["5"]
assert http.expected_http_body_size(h, False, "GET", 200) == 5
# no length
h = odict.ODictCaseless()
assert http.expected_http_body_size(h, False, "GET", 200) == -1
# no length request
h = odict.ODictCaseless()
assert http.expected_http_body_size(h, True, "GET", None) == 0
def test_expected_http_body_size():
# gibber in the content-length field
h = odict.ODictCaseless()
h["content-length"] = ["foo"]
assert http.expected_http_body_size(h, False, "GET", 200) is None
# negative number in the content-length field
h = odict.ODictCaseless()
h["content-length"] = ["-7"]
assert http.expected_http_body_size(h, False, "GET", 200) is None
# explicit length
h = odict.ODictCaseless()
h["content-length"] = ["5"]
assert http.expected_http_body_size(h, False, "GET", 200) == 5
# no length
h = odict.ODictCaseless()
assert http.expected_http_body_size(h, False, "GET", 200) == -1
# no length request
h = odict.ODictCaseless()
assert http.expected_http_body_size(h, True, "GET", None) == 0
def check_close_connection(self, flow):
"""
Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so.
"""
close_connection = (
http.connection_close(flow.request.httpversion, flow.request.headers) or
http.connection_close(flow.response.httpversion, flow.response.headers) or
http.expected_http_body_size(flow.response.headers, False, flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return True
return False
def check_close_connection(self, flow):
"""
Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so.
"""
close_connection = (http.connection_close(flow.request.httpversion,
flow.request.headers)
or http.connection_close(flow.response.httpversion,
flow.response.headers)
or http.expected_http_body_size(
flow.response.headers, False,
flow.request.method, flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return True
return False
def handle_flow(self):
flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live)
try:
req = HTTPRequest.from_stream(self.c.client_conn.rfile,
body_size_limit=self.c.config.body_size_limit)
self.c.log("request", "debug", [req._assemble_first_line(req.form_in)])
send_request_upstream = self.process_request(flow, req)
if not send_request_upstream:
return True
# Be careful NOT to assign the request to the flow before
# process_request completes. This is because the call can raise an
# exception. If the request object is already attached, this results
# in an Error object that has an attached request that has not been
# sent through to the Master.
flow.request = req
request_reply = self.c.channel.ask("request", flow.request)
self.determine_server_address(flow, flow.request)
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow
if request_reply is None or request_reply == KILL:
return False
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
else:
# read initially in "stream" mode, so we can get the headers separately
flow.response = self.get_response_from_server(flow.request, include_body=False)
# call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True
self.c.channel.ask("responseheaders", flow.response)
# now get the rest of the request body, if body still needs to be read but not streaming this response
if flow.response.stream:
flow.response.content = CONTENT_MISSING
else:
flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit,
flow.request.method, flow.response.code, False)
# no further manipulation of self.c.server_conn beyond this point
# we can safely set it as the final attribute value here.
flow.server_conn = self.c.server_conn
self.c.log("response", "debug", [flow.response._assemble_first_line()])
response_reply = self.c.channel.ask("response", flow.response)
if response_reply is None or response_reply == KILL:
return False
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can send it to the client straight away.
self.c.client_conn.send(flow.response._assemble())
else:
# streaming:
# First send the body and then transfer the response incrementally:
h = flow.response._assemble_head(preserve_transfer_encoding=True)
self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False, 4096):
for part in chunk:
self.c.client_conn.wfile.write(part)
self.c.client_conn.wfile.flush()
flow.response.timestamp_end = utils.timestamp()
flow.timestamp_end = utils.timestamp()
close_connection = (
http.connection_close(flow.request.httpversion, flow.request.headers) or
http.connection_close(flow.response.httpversion, flow.response.headers) or
http.expected_http_body_size(flow.response.headers, False, flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return False
if flow.request.form_in == "authority" and flow.response.code == 200:
# TODO: Eventually add headers (space/usefulness tradeoff)
# Make sure to add state info before the actual upgrade happens.
# During the upgrade, we may receive an SNI indication from the client,
# which resets the upstream connection. If this is the case, we must
# already re-issue the CONNECT request at this point.
self.c.server_conn.state.append(("http", {"state": "connect",
"host": flow.request.host,
"port": flow.request.port}))
self.ssl_upgrade()
# If the user has changed the target server on this connection,
# restore the original target server
flow.live.restore_server()
flow.live = None
return True
except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e:
self.handle_error(e, flow)
def handle_flow(self):
flow = HTTPFlow(self.c.client_conn, self.c.server_conn,
self.change_server)
try:
req = HTTPRequest.from_stream(
self.c.client_conn.rfile,
body_size_limit=self.c.config.body_size_limit)
self.c.log("request", "debug",
[req._assemble_first_line(req.form_in)])
send_request_upstream = self.process_request(flow, req)
if not send_request_upstream:
return True
# Be careful NOT to assign the request to the flow before
# process_request completes. This is because the call can raise an
# exception. If the request object is already attached, this results
# in an Error object that has an attached request that has not been
# sent through to the Master.
flow.request = req
request_reply = self.c.channel.ask("request", flow.request)
self.determine_server_address(flow, flow.request)
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow
if request_reply is None or request_reply == KILL:
return False
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
else:
# read initially in "stream" mode, so we can get the headers separately
flow.response = self.get_response_from_server(
flow.request, include_body=False)
# call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True
self.c.channel.ask("responseheaders", flow.response)
# now get the rest of the request body, if body still needs to be read but not streaming this response
if flow.response.stream:
flow.response.content = CONTENT_MISSING
else:
flow.response.content = http.read_http_body(
self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False)
# no further manipulation of self.c.server_conn beyond this point
# we can safely set it as the final attribute value here.
flow.server_conn = self.c.server_conn
self.c.log("response", "debug",
[flow.response._assemble_first_line()])
response_reply = self.c.channel.ask("response", flow.response)
if response_reply is None or response_reply == KILL:
return False
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can send it to the client straight away.
self.c.client_conn.send(flow.response._assemble())
else:
# streaming:
# First send the body and then transfer the response incrementally:
h = flow.response._assemble_head(
preserve_transfer_encoding=True)
self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(
self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False, 4096):
for part in chunk:
self.c.client_conn.wfile.write(part)
self.c.client_conn.wfile.flush()
flow.timestamp_end = utils.timestamp()
close_connection = (
http.connection_close(flow.request.httpversion,
flow.request.headers)
or http.connection_close(flow.response.httpversion,
flow.response.headers)
or http.expected_http_body_size(flow.response.headers, False,
flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return False
if flow.request.form_in == "authority" and flow.response.code == 200:
# TODO: Eventually add headers (space/usefulness tradeoff)
# Make sure to add state info before the actual upgrade happens.
# During the upgrade, we may receive an SNI indication from the client,
# which resets the upstream connection. If this is the case, we must
# already re-issue the CONNECT request at this point.
self.c.server_conn.state.append(("http", {
"state": "connect",
"host": flow.request.host,
"port": flow.request.port
}))
self.ssl_upgrade()
# If the user has changed the target server on this connection,
# restore the original target server
self.restore_server()
return True
except (HttpAuthenticationError, http.HttpError, proxy.ProxyError,
tcp.NetLibError), e:
self.handle_error(e, flow)
