def test_stream_chunked(self):
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connection.connect(("127.0.0.1", self.proxy.port))
fconn = connection.makefile()
spec = '200:h"Transfer-Encoding"="chunked":r:b"4\\r\\nthis\\r\\n7\\r\\nisatest\\r\\n0\\r\\n\\r\\n"'
connection.send(
"GET %s/p/%s HTTP/1.1\r\n" %
(self.server.urlbase, spec))
connection.send("\r\n")
httpversion, code, msg, headers, content = http.read_response(
fconn, "GET", None, include_body=False)
assert headers["Transfer-Encoding"][0] == 'chunked'
assert code == 200
chunks = list(
content for _,
content,
_ in http.read_http_body_chunked(
fconn,
headers,
None,
"GET",
200,
False))
assert chunks == ["this", "isatest", ""]
connection.close()
def send_response_to_client(self, flow):
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can send it to the client straight away.
self.c.client_conn.send(flow.response.assemble())
else:
# streaming:
# First send the headers and then transfer the response incrementally:
h = flow.response._assemble_head(preserve_transfer_encoding=True)
self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False, 4096):
for part in chunk:
self.c.client_conn.wfile.write(part)
self.c.client_conn.wfile.flush()
flow.response.timestamp_end = utils.timestamp()
def send_response_to_client(self, flow):
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can send it to the client straight away.
self.c.client_conn.send(flow.response.assemble())
else:
# streaming:
# First send the body and then transfer the response incrementally:
h = flow.response._assemble_head(preserve_transfer_encoding=True)
self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(
self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False, 4096):
for part in chunk:
self.c.client_conn.wfile.write(part)
self.c.client_conn.wfile.flush()
flow.response.timestamp_end = utils.timestamp()
def test_stream_chunked(self):
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connection.connect(("127.0.0.1", self.proxy.port))
fconn = connection.makefile()
spec = '200:h"Transfer-Encoding"="chunked":r:b"4\\r\\nthis\\r\\n7\\r\\nisatest\\r\\n0\\r\\n\\r\\n"'
connection.send("GET %s/p/%s HTTP/1.1\r\n"%(self.server.urlbase, spec))
connection.send("\r\n")
httpversion, code, msg, headers, content = http.read_response(fconn, "GET", None, include_body=False)
assert headers["Transfer-Encoding"][0] == 'chunked'
assert code == 200
chunks = list(content for _, content, _ in http.read_http_body_chunked(fconn, headers, None, "GET", 200, False))
assert chunks == ["this", "isatest", ""]
connection.close()
def handle_flow(self):
flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live)
try:
req = HTTPRequest.from_stream(self.c.client_conn.rfile,
body_size_limit=self.c.config.body_size_limit)
self.c.log("request", "debug", [req._assemble_first_line(req.form_in)])
send_request_upstream = self.process_request(flow, req)
if not send_request_upstream:
return True
# Be careful NOT to assign the request to the flow before
# process_request completes. This is because the call can raise an
# exception. If the request object is already attached, this results
# in an Error object that has an attached request that has not been
# sent through to the Master.
flow.request = req
request_reply = self.c.channel.ask("request", flow.request)
self.determine_server_address(flow, flow.request)
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow
if request_reply is None or request_reply == KILL:
return False
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
else:
# read initially in "stream" mode, so we can get the headers separately
flow.response = self.get_response_from_server(flow.request, include_body=False)
# call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True
self.c.channel.ask("responseheaders", flow.response)
# now get the rest of the request body, if body still needs to be read but not streaming this response
if flow.response.stream:
flow.response.content = CONTENT_MISSING
else:
flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit,
flow.request.method, flow.response.code, False)
# no further manipulation of self.c.server_conn beyond this point
# we can safely set it as the final attribute value here.
flow.server_conn = self.c.server_conn
self.c.log("response", "debug", [flow.response._assemble_first_line()])
response_reply = self.c.channel.ask("response", flow.response)
if response_reply is None or response_reply == KILL:
return False
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can send it to the client straight away.
self.c.client_conn.send(flow.response._assemble())
else:
# streaming:
# First send the body and then transfer the response incrementally:
h = flow.response._assemble_head(preserve_transfer_encoding=True)
self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False, 4096):
for part in chunk:
self.c.client_conn.wfile.write(part)
self.c.client_conn.wfile.flush()
flow.response.timestamp_end = utils.timestamp()
flow.timestamp_end = utils.timestamp()
close_connection = (
http.connection_close(flow.request.httpversion, flow.request.headers) or
http.connection_close(flow.response.httpversion, flow.response.headers) or
http.expected_http_body_size(flow.response.headers, False, flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return False
if flow.request.form_in == "authority" and flow.response.code == 200:
# TODO: Eventually add headers (space/usefulness tradeoff)
# Make sure to add state info before the actual upgrade happens.
# During the upgrade, we may receive an SNI indication from the client,
# which resets the upstream connection. If this is the case, we must
# already re-issue the CONNECT request at this point.
self.c.server_conn.state.append(("http", {"state": "connect",
"host": flow.request.host,
"port": flow.request.port}))
self.ssl_upgrade()
# If the user has changed the target server on this connection,
# restore the original target server
flow.live.restore_server()
flow.live = None
return True
except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e:
self.handle_error(e, flow)
def handle_flow(self):
flow = HTTPFlow(self.c.client_conn, self.c.server_conn,
self.change_server)
try:
req = HTTPRequest.from_stream(
self.c.client_conn.rfile,
body_size_limit=self.c.config.body_size_limit)
self.c.log("request", "debug",
[req._assemble_first_line(req.form_in)])
send_request_upstream = self.process_request(flow, req)
if not send_request_upstream:
return True
# Be careful NOT to assign the request to the flow before
# process_request completes. This is because the call can raise an
# exception. If the request object is already attached, this results
# in an Error object that has an attached request that has not been
# sent through to the Master.
flow.request = req
request_reply = self.c.channel.ask("request", flow.request)
self.determine_server_address(flow, flow.request)
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow
if request_reply is None or request_reply == KILL:
return False
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
else:
# read initially in "stream" mode, so we can get the headers separately
flow.response = self.get_response_from_server(
flow.request, include_body=False)
# call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True
self.c.channel.ask("responseheaders", flow.response)
# now get the rest of the request body, if body still needs to be read but not streaming this response
if flow.response.stream:
flow.response.content = CONTENT_MISSING
else:
flow.response.content = http.read_http_body(
self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False)
# no further manipulation of self.c.server_conn beyond this point
# we can safely set it as the final attribute value here.
flow.server_conn = self.c.server_conn
self.c.log("response", "debug",
[flow.response._assemble_first_line()])
response_reply = self.c.channel.ask("response", flow.response)
if response_reply is None or response_reply == KILL:
return False
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can send it to the client straight away.
self.c.client_conn.send(flow.response._assemble())
else:
# streaming:
# First send the body and then transfer the response incrementally:
h = flow.response._assemble_head(
preserve_transfer_encoding=True)
self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(
self.c.server_conn.rfile, flow.response.headers,
self.c.config.body_size_limit, flow.request.method,
flow.response.code, False, 4096):
for part in chunk:
self.c.client_conn.wfile.write(part)
self.c.client_conn.wfile.flush()
flow.timestamp_end = utils.timestamp()
close_connection = (
http.connection_close(flow.request.httpversion,
flow.request.headers)
or http.connection_close(flow.response.httpversion,
flow.response.headers)
or http.expected_http_body_size(flow.response.headers, False,
flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
pass
else:
return False
if flow.request.form_in == "authority" and flow.response.code == 200:
# TODO: Eventually add headers (space/usefulness tradeoff)
# Make sure to add state info before the actual upgrade happens.
# During the upgrade, we may receive an SNI indication from the client,
# which resets the upstream connection. If this is the case, we must
# already re-issue the CONNECT request at this point.
self.c.server_conn.state.append(("http", {
"state": "connect",
"host": flow.request.host,
"port": flow.request.port
}))
self.ssl_upgrade()
# If the user has changed the target server on this connection,
# restore the original target server
self.restore_server()
return True
except (HttpAuthenticationError, http.HttpError, proxy.ProxyError,
tcp.NetLibError), e:
self.handle_error(e, flow)
