def start_response(status, headers, exc_info=None):
if exc_info:
if state["headers_sent"]:
raise exc_info[1]
elif state["status"]:
raise AssertionError('Response already started')
state["status"] = status
state["headers"] = http.Headers([[strutils.always_bytes(k), strutils.always_bytes(v)] for k, v in headers])
if exc_info:
self.error_page(soc, state["headers_sent"], traceback.format_tb(exc_info[2]))
state["headers_sent"] = True
def start_response(status, headers, exc_info=None):
if exc_info:
if state["headers_sent"]:
raise exc_info[1]
elif state["status"]:
raise AssertionError('Response already started')
state["status"] = status
state["headers"] = http.Headers([[strutils.always_bytes(k), strutils.always_bytes(v)] for k, v in headers])
if exc_info:
self.error_page(soc, state["headers_sent"], traceback.format_tb(exc_info[2]))
state["headers_sent"] = True
def make(
cls,
method: str,
url: str,
content: Union[bytes, str] = "",
headers: Union[Headers, Dict[Union[str, bytes], Union[str, bytes]],
Iterable[Tuple[bytes, bytes]]] = ()
) -> "Request":
"""
Simplified API for creating request objects.
"""
# Headers can be list or dict, we differentiate here.
if isinstance(headers, Headers):
pass
elif isinstance(headers, dict):
headers = Headers((always_bytes(k, "utf-8", "surrogateescape"),
always_bytes(v, "utf-8", "surrogateescape"))
for k, v in headers.items())
elif isinstance(headers, Iterable):
headers = Headers(headers)
else:
raise TypeError(
"Expected headers to be an iterable or dict, but is {}.".
format(type(headers).__name__))
req = cls(
"",
0,
method.encode("utf-8", "surrogateescape"),
b"",
b"",
b"",
b"HTTP/1.1",
headers,
b"",
None,
time.time(),
time.time(),
)
req.url = url
# Assign this manually to update the content-length header.
if isinstance(content, bytes):
req.content = content
elif isinstance(content, str):
req.text = content
else:
raise TypeError(
f"Expected content to be str or bytes, but is {type(content).__name__}."
)
return req
def parse_webform(flow):
r = flow.request
_, netloc, path, _, query, _ = urllib.parse.urlparse(r.url)
# these parameters will be ignored during matching.
to_bytes = lambda params: [strutils.always_bytes(i) for i in params]
queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
headers = dict(r.data.headers)
form_items = []
# matches json-type responses. for example,
# application/json;charset=UTF-8
# application/xxx-json.
content_type = (headers.get('content-type', '')
or headers.get('Content-Type', ''))
if re.match(r'application/.*json.*', content_type):
form_unsored = json.loads(r.data.content.decode('utf-8'))
form_items = [(k, json.dumps(form_unsored[k], sort_keys=True))
for k in sorted(form_unsored.keys())]
else:
form_contents = r.urlencoded_form or r.multipart_form
log_warn('form contents = %s', form_contents)
if form_contents:
form_items = form_contents.items(multi=True)
form = dict(form_items)
return form
def parse_upstream_auth(auth):
pattern = re.compile(".+:")
if pattern.search(auth) is None:
raise exceptions.OptionsError(
"Invalid upstream auth specification: %s" % auth
)
return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
def parse_upstream_auth(auth):
pattern = re.compile(".+:")
if pattern.search(auth) is None:
raise exceptions.OptionsError(
"Invalid upstream auth specification: %s" % auth
)
return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
def make(
cls,
status_code: int = 200,
content: Union[bytes, str] = b"",
headers: Union[Headers, Mapping[str, Union[str, bytes]],
Iterable[Tuple[bytes, bytes]]] = ()
) -> "Response":
"""
Simplified API for creating response objects.
"""
if isinstance(headers, Headers):
headers = headers
elif isinstance(headers, dict):
headers = Headers((
always_bytes(k, "utf-8", "surrogateescape"), # type: ignore
always_bytes(v, "utf-8", "surrogateescape"))
for k, v in headers.items())
elif isinstance(headers, Iterable):
headers = Headers(headers) # type: ignore
else:
raise TypeError(
"Expected headers to be an iterable or dict, but is {}.".
format(type(headers).__name__))
resp = cls(
b"HTTP/1.1",
status_code,
status_codes.RESPONSES.get(status_code, "").encode(),
headers,
None,
None,
time.time(),
time.time(),
)
# Assign this manually to update the content-length header.
if isinstance(content, bytes):
resp.content = content
elif isinstance(content, str):
resp.text = content
else:
raise TypeError(
f"Expected content to be str or bytes, but is {type(content).__name__}."
)
return resp
def test_always_bytes():
assert strutils.always_bytes(bytes(range(256))) == bytes(range(256))
assert strutils.always_bytes("foo") == b"foo"
with tutils.raises(ValueError):
strutils.always_bytes(u"\u2605", "ascii")
with tutils.raises(TypeError):
strutils.always_bytes(42, "ascii")
def test_always_bytes():
assert strutils.always_bytes(bytes(range(256))) == bytes(range(256))
assert strutils.always_bytes("foo") == b"foo"
with pytest.raises(ValueError):
strutils.always_bytes(u"\u2605", "ascii")
with pytest.raises(TypeError):
strutils.always_bytes(42, "ascii")
def values(self, settings):
if self.body:
bodygen = self.body.value.get_generator(settings)
length = len(self.body.value.get_generator(settings))
elif self.rawbody:
bodygen = self.rawbody.value.get_generator(settings)
length = len(self.rawbody.value.get_generator(settings))
elif self.nested_frame:
bodygen = NESTED_LEADER + strutils.always_bytes(self.nested_frame.parsed.spec())
length = len(bodygen)
else:
bodygen = None
length = 0
if self.toklength:
length = int(self.toklength.value)
frameparts = dict(
payload_length=length
)
if self.mask and self.mask.value:
frameparts["mask"] = True
if self.knone:
frameparts["masking_key"] = None
elif self.key:
key = self.key.values(settings)[0][:]
frameparts["masking_key"] = key
for i in ["opcode", "fin", "rsv1", "rsv2", "rsv3", "mask"]:
v = getattr(self, i, None)
if v is not None:
frameparts[i] = v.value
# import wsproto.frame_protocol
# wsproto.frame_protocol.Frame(
# opcode=frameparts["opcode"],
# payload=None,
# frame_finished=frameparts["fin"]
# )
frame = websockets_frame.FrameHeader(**frameparts)
vals = [bytes(frame)]
if bodygen:
if frame.masking_key and not self.rawbody:
masker = websockets_frame.Masker(frame.masking_key)
vals.append(
generators.TransformGenerator(
bodygen,
masker.mask
)
)
else:
vals.append(bodygen)
return vals
def _hash(self, flow):
"""
Calculates a loose hash of the flow request.
"""
r = flow.request
_, _, path, _, query, _ = urllib.parse.urlparse(r.url)
queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
key = [str(r.port), str(r.scheme), str(r.method), str(path)]
if not self.options.server_replay_ignore_content:
form_contents = r.urlencoded_form or r.multipart_form
if self.options.server_replay_ignore_payload_params and form_contents:
params = [
strutils.always_bytes(i)
for i in self.options.server_replay_ignore_payload_params
]
for p in form_contents.items(multi=True):
if p[0] not in params:
key.append(p)
else:
key.append(str(r.raw_content))
if not self.options.server_replay_ignore_host:
key.append(r.host)
filtered = []
ignore_params = self.options.server_replay_ignore_params or []
for p in queriesArray:
if p[0] not in ignore_params:
filtered.append(p)
for p in filtered:
key.append(p[0])
key.append(p[1])
if self.options.server_replay_use_headers:
headers = []
for i in self.options.server_replay_use_headers:
v = r.headers.get(i)
headers.append((i, v))
key.append(headers)
return hashlib.sha256(
repr(key).encode("utf8", "surrogateescape")
).digest()
def values(self, settings):
if self.body:
bodygen = self.body.value.get_generator(settings)
length = len(self.body.value.get_generator(settings))
elif self.rawbody:
bodygen = self.rawbody.value.get_generator(settings)
length = len(self.rawbody.value.get_generator(settings))
elif self.nested_frame:
bodygen = NESTED_LEADER + strutils.always_bytes(self.nested_frame.parsed.spec())
length = len(bodygen)
else:
bodygen = None
length = 0
if self.toklength:
length = int(self.toklength.value)
frameparts = dict(
payload_length=length
)
if self.mask and self.mask.value:
frameparts["mask"] = True
if self.knone:
frameparts["masking_key"] = None
elif self.key:
key = self.key.values(settings)[0][:]
frameparts["masking_key"] = key
for i in ["opcode", "fin", "rsv1", "rsv2", "rsv3", "mask"]:
v = getattr(self, i, None)
if v is not None:
frameparts[i] = v.value
frame = mitmproxy.net.websockets.FrameHeader(**frameparts)
vals = [bytes(frame)]
if bodygen:
if frame.masking_key and not self.rawbody:
masker = mitmproxy.net.websockets.Masker(frame.masking_key)
vals.append(
generators.TransformGenerator(
bodygen,
masker.mask
)
)
else:
vals.append(bodygen)
return vals
def _hash(self, flow):
"""
Calculates a loose hash of the flow request.
"""
r = flow.request
_, _, path, _, query, _ = urllib.parse.urlparse(r.url)
queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
key = [str(r.port), str(r.scheme), str(r.method), str(path)]
if not self.options.server_replay_ignore_content:
form_contents = r.urlencoded_form or r.multipart_form
if self.options.server_replay_ignore_payload_params and form_contents:
params = [
strutils.always_bytes(i)
for i in self.options.server_replay_ignore_payload_params
]
for p in form_contents.items(multi=True):
if p[0] not in params:
key.append(p)
else:
key.append(str(r.raw_content))
if not self.options.server_replay_ignore_host:
key.append(r.host)
filtered = []
ignore_params = self.options.server_replay_ignore_params or []
for p in queriesArray:
if p[0] not in ignore_params:
filtered.append(p)
for p in filtered:
key.append(p[0])
key.append(p[1])
if self.options.server_replay_use_headers:
headers = []
for i in self.options.server_replay_use_headers:
v = r.headers.get(i)
headers.append((i, v))
key.append(headers)
return hashlib.sha256(repr(key).encode("utf8",
"surrogateescape")).digest()
def method(self, method):
self.data.method = strutils.always_bytes(method, "utf-8", "surrogateescape")
def scheme(self, val: Union[str, bytes]) -> None:
self.data.scheme = always_bytes(val, "utf-8", "surrogateescape")
def path(self, val: Union[str, bytes]) -> None:
self.data.path = always_bytes(val, "utf-8", "surrogateescape")
def _hash(self, flow):
''' Calculates a loose hash of the flow request. '''
r = flow.request
_, netloc, path, _, query, _ = urllib.parse.urlparse(r.url)
# these parameters will be ignored during matching.
to_bytes = lambda params: [strutils.always_bytes(i) for i in params]
ignore_site_params = IGNORE_SITE_PARAMS.get(netloc, [])
ignore_url_params = IGNORE_URL_PARAMS.get(netloc + path, [])
whitelist_url_params = WHITELIST_URL_PARAMS.get(netloc + path, None)
queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
key = [str(r.port), str(r.scheme), str(r.method), str(path)]
headers = dict(r.data.headers)
form_contents = r.urlencoded_form or r.multipart_form
form_items = []
if re.match(r'application/.*json', headers.get('content-type', '')):
form_unsorted = json.loads(r.data.content.decode('utf-8'))
if isinstance(form_unsorted, list): # make sure form is a dict.
form_unsorted = {'data': form_unsorted}
form_items = [(k, json.dumps(form_unsorted[k], sort_keys=True))
for k in sorted(form_unsorted.keys())]
elif form_contents:
form_items = form_contents.items(multi=True)
if form_items:
for p in form_items:
param = strutils.always_bytes(p[0])
if param in to_bytes(ignore_site_params) or param in to_bytes(
ignore_url_params):
logger.warn('[proxy hash] ignoring parameter %s',
str(p[0]))
continue
if whitelist_url_params and param not in to_bytes(
whitelist_url_params):
logger.warn('[proxy hash] ignoring parameter %s',
str(p[0]))
continue
key.append(p)
else:
key.append(str(r.raw_content))
if not self.options.server_replay_ignore_host:
key.append(r.host)
filtered = []
for p in queriesArray:
param = p[0]
if param in ignore_site_params or param in ignore_url_params:
logger.warn('[proxy hash] ignoring parameter %s', str(p[0]))
continue
if whitelist_url_params and param not in whitelist_url_params:
logger.warn('[proxy hash] ignoring parameter %s', str(p[0]))
continue
filtered.append(p)
for p in filtered:
key.append(p[0])
key.append(p[1])
if self.options.server_replay_use_headers:
headers = []
for i in self.options.server_replay_use_headers:
v = r.headers.get(i)
headers.append((i, v))
key.append(headers)
binary = hashlib.sha256(repr(key).encode("utf8",
"surrogateescape")).digest()
if r.method == 'POST':
logger.warn('\033[91m Args %s \033[0m',
repr(key).encode("utf8", "surrogateescape"))
return str(base64.b64encode(binary), 'utf8')
def method(self, val: Union[str, bytes]) -> None:
self.data.method = always_bytes(val, "utf-8", "surrogateescape")
def method(self, method):
self.data.method = strutils.always_bytes(method, "utf-8",
"surrogateescape")
def http_version(self, http_version):
self.data.http_version = strutils.always_bytes(http_version, "utf-8", "surrogateescape")
def create_server_nonce(client_nonce):
return base64.b64encode(
hashlib.sha1(strutils.always_bytes(client_nonce) + MAGIC).digest())
def reason(self, reason: Union[str, bytes]) -> None:
self.data.reason = strutils.always_bytes(reason, "ISO-8859-1")
def http_version(self, http_version):
self.data.http_version = strutils.always_bytes(http_version, "utf-8",
"surrogateescape")
def reason(self, reason):
self.data.reason = strutils.always_bytes(reason, "ISO-8859-1",
"surrogateescape")
def reason(self, reason):
self.data.reason = strutils.always_bytes(reason, "ISO-8859-1", "surrogateescape")
def path(self, path):
self.data.path = strutils.always_bytes(path, "utf-8",
"surrogateescape")
def create_server_nonce(client_nonce):
return base64.b64encode(hashlib.sha1(strutils.always_bytes(client_nonce) + MAGIC).digest())
def scheme(self, scheme):
self.data.scheme = strutils.always_bytes(scheme, "utf-8", "surrogateescape")
def http_version(self, http_version: Union[str, bytes]) -> None:
self.data.http_version = strutils.always_bytes(http_version, "utf-8",
"surrogateescape")
def path(self, path):
self.data.path = strutils.always_bytes(path, "utf-8", "surrogateescape")
def _always_bytes(x):
return strutils.always_bytes(x, "utf-8", "surrogateescape")
def scheme(self, scheme):
self.data.scheme = strutils.always_bytes(scheme, "utf-8",
"surrogateescape")
def _always_bytes(x):
return strutils.always_bytes(x, "utf-8", "surrogateescape")
def _always_bytes(x: Union[str, bytes]) -> bytes:
return strutils.always_bytes(x, "utf-8", "surrogateescape")
def flowdetails(state, flow: mitmproxy.flow.Flow):
text = []
sc = flow.server_conn
cc = flow.client_conn
req: typing.Optional[http.HTTPRequest]
resp: typing.Optional[http.HTTPResponse]
if isinstance(flow, http.HTTPFlow):
req = flow.request
resp = flow.response
else:
req = None
resp = None
metadata = flow.metadata
if metadata is not None and len(metadata) > 0:
parts = [(str(k), repr(v)) for k, v in metadata.items()]
text.append(urwid.Text([("head", "Metadata:")]))
text.extend(common.format_keyvals(parts, indent=4))
if sc is not None and sc.ip_address:
text.append(urwid.Text([("head", "Server Connection:")]))
parts = [
("Address", human.format_address(sc.address)),
]
if sc.ip_address:
parts.append(
("Resolved Address", human.format_address(sc.ip_address)))
if resp:
parts.append(("HTTP Version", resp.http_version))
if sc.alpn_proto_negotiated:
parts.append(
("ALPN",
strutils.bytes_to_escaped_str(sc.alpn_proto_negotiated)))
text.extend(common.format_keyvals(parts, indent=4))
c = sc.cert
if c:
text.append(urwid.Text([("head", "Server Certificate:")]))
parts = [("Type", "%s, %s bits" % c.keyinfo),
("SHA1 digest", c.digest("sha1")),
("Valid to", str(c.notafter)),
("Valid from", str(c.notbefore)),
("Serial", str(c.serial)),
("Subject",
urwid.BoxAdapter(
urwid.ListBox(
common.format_keyvals(c.subject,
key_format="highlight")),
len(c.subject))),
("Issuer",
urwid.BoxAdapter(
urwid.ListBox(
common.format_keyvals(c.issuer,
key_format="highlight")),
len(c.issuer)))]
if c.altnames:
parts.append(("Alt names", ", ".join(
strutils.bytes_to_escaped_str(x) for x in c.altnames)))
text.extend(common.format_keyvals(parts, indent=4))
if cc is not None:
text.append(urwid.Text([("head", "Client Connection:")]))
parts = [
("Address", "{}:{}".format(cc.address[0], cc.address[1])),
]
if req:
parts.append(("HTTP Version", req.http_version))
if cc.tls_version:
parts.append(("TLS Version", cc.tls_version))
if cc.sni:
parts.append(("Server Name Indication",
strutils.bytes_to_escaped_str(
strutils.always_bytes(cc.sni, "idna"))))
if cc.cipher_name:
parts.append(("Cipher Name", cc.cipher_name))
if cc.alpn_proto_negotiated:
parts.append(
("ALPN",
strutils.bytes_to_escaped_str(cc.alpn_proto_negotiated)))
text.extend(common.format_keyvals(parts, indent=4))
parts = []
if cc is not None and cc.timestamp_start:
parts.append(("Client conn. established",
maybe_timestamp(cc, "timestamp_start")))
if cc.tls_established:
parts.append(("Client conn. TLS handshake",
maybe_timestamp(cc, "timestamp_tls_setup")))
parts.append(
("Client conn. closed", maybe_timestamp(cc, "timestamp_end")))
if sc is not None and sc.timestamp_start:
parts.append(
("Server conn. initiated", maybe_timestamp(sc, "timestamp_start")))
parts.append(("Server conn. TCP handshake",
maybe_timestamp(sc, "timestamp_tcp_setup")))
if sc.tls_established:
parts.append(("Server conn. TLS handshake",
maybe_timestamp(sc, "timestamp_tls_setup")))
parts.append(
("Server conn. closed", maybe_timestamp(sc, "timestamp_end")))
if req is not None and req.timestamp_start:
parts.append(
("First request byte", maybe_timestamp(req, "timestamp_start")))
parts.append(
("Request complete", maybe_timestamp(req, "timestamp_end")))
if resp is not None and resp.timestamp_start:
parts.append(
("First response byte", maybe_timestamp(resp, "timestamp_start")))
parts.append(
("Response complete", maybe_timestamp(resp, "timestamp_end")))
if parts:
# sort operations by timestamp
parts = sorted(parts, key=lambda p: p[1])
text.append(urwid.Text([("head", "Timing:")]))
text.extend(common.format_keyvals(parts, indent=4))
return searchable.Searchable(text)
