def clean_multipart_form(self, req: HTTPRequest) -> None:
"""Replace any problematic values of multipart form keys with constant data
Args:
req (HTTPRequest): The request to modify
"""
if req.multipart_form and self.form_keys:
updated_multipart_form_data = []
for key, val in req.multipart_form.items(multi=True):
if key in self.form_keys:
updated_multipart_form_data.append((key, self.constant))
else:
updated_multipart_form_data.append((key, val))
req._set_multipart_form(updated_multipart_form_data)
def clean_urlencoded_form(self, req: HTTPRequest) -> None:
'''Replace any problematic values of urlencoded form keys with constant data
Args:
req (HTTPRequest): The request to modify
'''
if req.urlencoded_form and self.form_keys:
updated_urlencoded_form_data = []
for key, val in req.urlencoded_form.items(multi=True):
if key in self.form_keys:
updated_urlencoded_form_data.append((key, self.constant))
else:
updated_urlencoded_form_data.append((key, val))
req._set_urlencoded_form(updated_urlencoded_form_data)
def modify_json_body(self, req: HTTPRequest, json_body: dict) -> None:
"""Modify the json body of a request by replacing any timestamp data with constant data
Args:
req (HTTPRequest): The request whose json body will be modified.
json_body (dict): The request body to modify.
"""
original_content = deepcopy(json_body)
modified = False
keys_to_replace = self.json_keys
logging.info('{}'.format(keys_to_replace))
for key_path in keys_to_replace:
body = json_body
keys = key_path.split('.')
logging.info('keypath parts: {}'.format(keys))
lastkey = keys[-1]
logging.info('lastkey: {}'.format(lastkey))
skip_key = False
for k in keys[:-1]:
if k in body:
body = body[k]
elif isinstance(body, list) and k.isdigit():
if int(k) > len(body) - 1:
skip_key = True
break
body = body[int(k)]
else:
skip_key = True
break
if not skip_key:
if lastkey in body:
logging.info('modifying request to "{}"'.format(
req.pretty_url))
body[lastkey] = self.constant
modified = True
elif isinstance(body, list) and lastkey.isdigit(
) and int(lastkey) <= len(body) - 1:
logging.info('modifying request to "{}"'.format(
req.pretty_url))
body[int(lastkey)] = self.constant
modified = True
if modified:
logging.info('original request body:\n{}'.format(
json.dumps(original_content, indent=4)))
logging.info('modified request body:\n{}'.format(
json.dumps(json_body, indent=4)))
req.set_content(json.dumps(json_body).encode())
def handle_url_query(self, req: HTTPRequest) -> None:
query_data = req._get_query()
logging.info('query_data: {}'.format(query_data))
for key, val in query_data:
# don't bother trying to interpret an argument less than 4 characters as some type of timestamp
if len(val) > 4:
if self.safely_parse(val):
self.query_keys.add(key)
def clean_url_query(self, req: HTTPRequest) -> None:
"""Replace any problematic values of query parameters with constant data
Args:
req (HTTPRequest): The request to modify
"""
query_data = sorted(req._get_query())
logging.info('fetched query_data: {}'.format(query_data))
updated_query_data = []
if query_data and self.query_keys:
for key, val in query_data:
if key in self.query_keys:
updated_query_data.append((key, self.constant))
else:
updated_query_data.append((key, val))
req._set_query(updated_query_data or query_data)
logging.info(f'updated query_data: {req._get_query()}')
def request_content_for_console(request: http.HTTPRequest) -> str:
try:
text = request.get_text(strict=True)
assert text
except ValueError:
# shlex.quote doesn't support a bytes object
# see https://github.com/python/cpython/pull/10871
raise exceptions.CommandError("Request content must be valid unicode")
escape_control_chars = {chr(i): f"\\x{i:02x}" for i in range(32)}
return "".join(escape_control_chars.get(x, x) for x in text)
def _load_http_request(o: http_pb2.HTTPRequest) -> HTTPRequest:
d: dict = {}
_move_attrs(o, d, ['host', 'port', 'method', 'scheme', 'authority', 'path', 'http_version', 'content',
'timestamp_start', 'timestamp_end'])
if d['content'] is None:
d['content'] = b""
d["headers"] = []
for header in o.headers:
d["headers"].append((bytes(header.name, "utf-8"), bytes(header.value, "utf-8")))
return HTTPRequest(**d)
def handle_url_query(self, req: HTTPRequest) -> None:
query_data = req._get_query()
print('query_data: {}'.format(query_data))
for key, val in query_data:
# don't bother trying to interpret an argument less than 4 characters as some type of timestamp
if len(val) > 4:
try:
parse(val)
self.query_keys.add(key)
except ValueError:
pass
def to_mitmproxy(self) -> HTTPRequest:
parsed_url = urlparse(self.url)
return HTTPRequest(
first_line_format='absolute',
method=self.method,
scheme=parsed_url.scheme,
host=parsed_url.hostname,
port=parsed_url.port or getservbyname(parsed_url.scheme),
path=parsed_url.path,
http_version=self.http_version,
headers=self.headers.to_mitmproxy(),
content=self.body,
)
def flow():
request = HTTPRequest(first_line_format='first_line',
host=b'localhost',
path=b'/test/',
http_version=b'1.1',
port=1234,
method=b'',
scheme=b'',
headers=Headers([(b"Host", b"example.com")]),
content=None,
timestamp_start=111.1)
flow = HTTPFlow(client_conn=MagicMock(), server_conn=MagicMock())
flow.request = request
return flow
'Connection': ['keep-alive'],
'User-Agent': ['python-requests/2.18.4'],
},
'method': 'GET',
'url': 'http://127.0.0.1/test',
}
TEST_MITM_REQUEST = HTTPRequest(
first_line_format='absolute',
method='GET',
scheme='http',
host='127.0.0.1',
port=80,
path='/test',
http_version='HTTP/1.1',
headers=[
(b'Content-Type', b'application/json; charset=UTF-8'),
(b'Accept-Encoding', b'gzip, deflate'),
(b'Connection', b'keep-alive'),
(b'User-Agent', b'python-requests/2.18.4'),
],
content=b'{"message": "Witaj, \xc5\x9bwiecie!"}'
)
TEST_MITM_REQUEST_GZIP = HTTPRequest(
first_line_format='absolute',
method='GET',
scheme='http',
host='127.0.0.1',
port=80,
def request(self, flow: http.HTTPRequest) -> None:
if self.mock_data.url in flow.request.url:
flow.response = http.HTTPResponse.make(
status_code=self.mock_data.status_code,
content=self.mock_data.body,
headers=self.mock_data.content_type)