def test_replace_simple(self):
headers = Headers(Host="example.com", Accept="text/plain")
replacements = headers.replace("Host: ", "X-Host: ")
assert replacements == 1
assert headers["X-Host"] == "example.com"
assert "Host" not in headers
assert headers["Accept"] == "text/plain"
def test_response_interceptor_called(self):
mock_flow = Mock()
mock_flow.request.id = '12345'
mock_flow.request.url = 'http://somewhere.com/some/path'
mock_flow.request.headers = Headers()
mock_flow.request.raw_content = b''
mock_flow.response.status_code = 200
mock_flow.response.reason = 'OK'
mock_flow.response.headers = Headers([(b'Content-Length', b'6')])
mock_flow.response.raw_content = b'foobar'
def intercept(req, res):
if req.url == 'http://somewhere.com/some/path':
res.status_code = 201
res.reason = 'Created'
res.headers['a'] = 'b'
res.body = b'foobarbaz'
self.server.response_interceptor = intercept
self.handler.response(mock_flow)
self.assertEqual(201, mock_flow.response.status_code)
self.assertEqual('Created', mock_flow.response.reason)
self.assertEqual({
'Content-Length': '6',
'a': 'b'
}, dict(mock_flow.response.headers))
self.assertEqual(b'foobarbaz', mock_flow.response.raw_content)
def test_replace_simple(self):
headers = Headers(Host="example.com", Accept="text/plain")
replacements = headers.replace("Host: ", "X-Host: ")
assert replacements == 1
assert headers["X-Host"] == "example.com"
assert "Host" not in headers
assert headers["Accept"] == "text/plain"
def __init__(
self,
http_version: bytes,
status_code: int,
reason: bytes,
headers: Union[Headers, Tuple[Tuple[bytes, bytes], ...]],
content: Optional[bytes],
trailers: Union[None, Headers, Tuple[Tuple[bytes, bytes], ...]],
timestamp_start: float,
timestamp_end: Optional[float],
):
# auto-convert invalid types to retain compatibility with older code.
if isinstance(http_version, str):
http_version = http_version.encode("ascii", "strict")
if isinstance(reason, str):
reason = reason.encode("ascii", "strict")
if isinstance(content, str):
raise ValueError("Content must be bytes, not {}".format(
type(content).__name__))
if not isinstance(headers, Headers):
headers = Headers(headers)
if trailers is not None and not isinstance(trailers, Headers):
trailers = Headers(trailers)
self.data = ResponseData(
http_version=http_version,
status_code=status_code,
reason=reason,
headers=headers,
content=content,
trailers=trailers,
timestamp_start=timestamp_start,
timestamp_end=timestamp_end,
)
def test_bytes(self):
headers = Headers(Host="example.com")
assert bytes(headers) == b"Host: example.com\r\n"
headers = Headers([[b"Host", b"example.com"],
[b"Accept", b"text/plain"]])
assert bytes(headers) == b"Host: example.com\r\nAccept: text/plain\r\n"
headers = Headers()
assert bytes(headers) == b""
def make(
cls,
method: str,
url: str,
content: Union[bytes, str] = "",
headers: Union[Headers, Dict[Union[str, bytes], Union[str, bytes]],
Iterable[Tuple[bytes, bytes]]] = ()
) -> "Request":
"""
Simplified API for creating request objects.
"""
# Headers can be list or dict, we differentiate here.
if isinstance(headers, Headers):
pass
elif isinstance(headers, dict):
headers = Headers((always_bytes(k, "utf-8", "surrogateescape"),
always_bytes(v, "utf-8", "surrogateescape"))
for k, v in headers.items())
elif isinstance(headers, Iterable):
headers = Headers(headers)
else:
raise TypeError(
"Expected headers to be an iterable or dict, but is {}.".
format(type(headers).__name__))
req = cls(
"",
0,
method.encode("utf-8", "surrogateescape"),
b"",
b"",
b"",
b"HTTP/1.1",
headers,
b"",
None,
time.time(),
time.time(),
)
req.url = url
# Assign this manually to update the content-length header.
if isinstance(content, bytes):
req.content = content
elif isinstance(content, str):
req.text = content
else:
raise TypeError(
f"Expected content to be str or bytes, but is {type(content).__name__}."
)
return req
def __init__(
self,
host: str,
port: int,
method: bytes,
scheme: bytes,
authority: bytes,
path: bytes,
http_version: bytes,
headers: Union[Headers, Tuple[Tuple[bytes, bytes], ...]],
content: Optional[bytes],
trailers: Union[None, Headers, Tuple[Tuple[bytes, bytes], ...]],
timestamp_start: float,
timestamp_end: Optional[float],
):
# auto-convert invalid types to retain compatibility with older code.
if isinstance(host, bytes):
host = host.decode("idna", "strict")
if isinstance(method, str):
method = method.encode("ascii", "strict")
if isinstance(scheme, str):
scheme = scheme.encode("ascii", "strict")
if isinstance(authority, str):
authority = authority.encode("ascii", "strict")
if isinstance(path, str):
path = path.encode("ascii", "strict")
if isinstance(http_version, str):
http_version = http_version.encode("ascii", "strict")
if isinstance(content, str):
raise ValueError(
f"Content must be bytes, not {type(content).__name__}")
if not isinstance(headers, Headers):
headers = Headers(headers)
if trailers is not None and not isinstance(trailers, Headers):
trailers = Headers(trailers)
self.data = RequestData(
host=host,
port=port,
method=method,
scheme=scheme,
authority=authority,
path=path,
http_version=http_version,
headers=headers,
content=content,
trailers=trailers,
timestamp_start=timestamp_start,
timestamp_end=timestamp_end,
)
def update_headers(request):
if self.custom_headers != []:
request.headers = Headers()
for key, value in self.custom_headers:
if key.get != "":
request.headers[key.get()] = value.get()
def test_capture_response_called(self):
mock_flow = Mock()
mock_flow.request.id = '12345'
mock_flow.request.url = 'http://somewhere.com/some/path'
mock_flow.response.status_code = 200
mock_flow.response.reason = 'OK'
mock_flow.response.headers = Headers([(b'Content-Length', b'6')])
mock_flow.response.raw_content = b'foobar'
captured_response = None
def capture_response(*args):
nonlocal captured_response
captured_response = args[2]
self.capture_response.side_effect = capture_response
self.handler.response(mock_flow)
self.capture_response.assert_called_once_with(
'12345', 'http://somewhere.com/some/path', ANY)
self.assertEqual(200, captured_response.status_code)
self.assertEqual('OK', captured_response.reason)
self.assertEqual({'Content-Length': '6'},
dict(captured_response.headers))
self.assertEqual(b'foobar', captured_response.body)
def test_capture_request_called(self):
mock_flow = Mock()
mock_flow.request.url = 'http://somewhere.com/some/path'
mock_flow.request.method = 'GET'
mock_flow.request.headers = Headers([(b'Accept-Encoding', b'identity')
])
mock_flow.request.raw_content = b'foobar'
captured_request = None
def capture_request(req):
nonlocal captured_request
req.id = '12345'
captured_request = req
self.capture_request.side_effect = capture_request
self.handler.request(mock_flow)
self.assertEqual(1, self.capture_request.call_count)
self.assertEqual('GET', captured_request.method)
self.assertEqual('http://somewhere.com/some/path',
captured_request.url)
self.assertEqual({'Accept-Encoding': 'identity'},
dict(captured_request.headers))
self.assertEqual(b'foobar', captured_request.body)
self.assertEqual('12345', captured_request.id)
self.assertEqual('12345', mock_flow.request.id)
def _2host(self):
return Headers(
(
(b"Host", b"example.com"),
(b"host", b"example.org")
)
)
def test_request_interceptor_called(self):
mock_flow = Mock()
mock_flow.request.url = 'http://somewhere.com/some/path'
mock_flow.request.method = 'GET'
mock_flow.request.headers = Headers([(b'Accept-Encoding', b'identity')
])
mock_flow.request.raw_content = b''
def intercept(req):
req.method = 'POST'
req.url = 'https://www.google.com/foo/bar?x=y'
req.body = b'foobarbaz'
req.headers['a'] = 'b'
self.server.request_interceptor = intercept
self.handler.request(mock_flow)
self.assertEqual('POST', mock_flow.request.method)
self.assertEqual('https://www.google.com/foo/bar?x=y',
mock_flow.request.url)
self.assertEqual({
'Accept-Encoding': 'identity',
'a': 'b'
}, dict(mock_flow.request.headers))
self.assertEqual(b'foobarbaz', mock_flow.request.raw_content)
def request(self, flow):
if (flow.request.pretty_host == prettyHost
and flow.request.method == reqMethod):
if (flow.request.pretty_url == prettyURL):
if self.process != None:
self.process.terminate()
flow.request.decode()
uForm = json.loads(flow.request.content)
quizMatch = self.regex.match(flow.request.headers['referer'])
quiz = quizMatch.group(1)
runDir = os.path.dirname(os.path.abspath(__file__))
runExec = os.path.join(runDir, 'MentiMemer')
ctx.log.info("Calling \'" + runExec + "\'")
self.process = Popen([
runExec, quiz, uForm['question'], uForm['question_type'],
json.dumps(uForm['vote'], ensure_ascii=False)
],
cwd=runDir)
elif (flow.request.pretty_host == stopHost):
if self.process != None:
self.process.terminate()
self.process = None
flow.response = HTTPResponse(
"HTTP/1.1", 200, "OK", Headers(Content_Type="text/html"),
b'<!DOCTYPE html><html><body><big style=\"font-size: 5em\">Stopped auto-send</big></body></html>'
)
def test_set(self):
headers = Headers()
headers[u"foo"] = u"1"
headers[b"bar"] = b"2"
headers["baz"] = b"3"
with pytest.raises(TypeError):
headers["foobar"] = 42
assert len(headers) == 3
def to_mitmproxy(self) -> Headers:
fields = []
for key, values in self.headers.items():
for value in values:
fields.append((key.encode('ascii'), value.encode('ascii')))
return Headers(fields=fields)
def make(
cls,
status_code: int = 200,
content: Union[bytes, str] = b"",
headers: Union[Headers, Dict[Union[str, bytes], Union[str, bytes]],
Iterable[Tuple[bytes, bytes]]] = ()
) -> "Response":
"""
Simplified API for creating response objects.
"""
if isinstance(headers, Headers):
headers = headers
elif isinstance(headers, dict):
headers = Headers((always_bytes(k, "utf-8", "surrogateescape"),
always_bytes(v, "utf-8", "surrogateescape"))
for k, v in headers.items())
elif isinstance(headers, Iterable):
headers = Headers(headers)
else:
raise TypeError(
"Expected headers to be an iterable or dict, but is {}.".
format(type(headers).__name__))
resp = cls(
b"HTTP/1.1",
status_code,
status_codes.RESPONSES.get(status_code, "").encode(),
headers,
None,
None,
time.time(),
time.time(),
)
# Assign this manually to update the content-length header.
if isinstance(content, bytes):
resp.content = content
elif isinstance(content, str):
resp.text = content
else:
raise TypeError(
f"Expected content to be str or bytes, but is {type(content).__name__}."
)
return resp
def test_request_modifier_called(self):
mock_flow = Mock()
mock_flow.request.url = 'http://somewhere.com/some/path'
mock_flow.request.method = 'GET'
mock_flow.request.headers = Headers([(b'Accept-Encoding', b'identity')
])
mock_flow.request.raw_content = b''
self.handler.request(mock_flow)
self.server.modifier.modify_request.assert_called_once_with(
mock_flow.request, bodyattr='raw_content')
def test_disable_encoding(self):
mock_flow = Mock()
mock_flow.request.url = 'http://somewhere.com/some/path'
mock_flow.request.method = 'GET'
mock_flow.request.headers = Headers([(b'Accept-Encoding', b'gzip')])
mock_flow.request.raw_content = b''
self.server.options['disable_encoding'] = True
self.handler.request(mock_flow)
self.assertEqual({'Accept-Encoding': 'identity'},
dict(mock_flow.request.headers))
def test_init(self):
headers = Headers()
assert len(headers) == 0
headers = Headers([[b"Host", b"example.com"]])
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(Host="example.com")
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(
[[b"Host", b"invalid"]],
Host="example.com"
)
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(
[[b"Host", b"invalid"], [b"Accept", b"text/plain"]],
Host="example.com"
)
assert len(headers) == 2
assert headers["Host"] == "example.com"
assert headers["Accept"] == "text/plain"
with pytest.raises(TypeError):
Headers([[b"Host", u"not-bytes"]])
def test_response_modifier_called(self):
mock_flow = Mock()
mock_flow.request.id = '12345'
mock_flow.request.url = 'http://somewhere.com/some/path'
mock_flow.response.status_code = 200
mock_flow.response.reason = 'OK'
mock_flow.response.headers = Headers([(b'Content-Length', b'6')])
mock_flow.response.raw_content = b'foobar'
self.handler.response(mock_flow)
self.server.modifier.modify_response.assert_called_once_with(
mock_flow.response, mock_flow.request)
def _replace_in_headers(headers: nheaders.Headers,
modifiers_dict: dict) -> int:
"""
Taken from original mitmproxy's Header class implementation with some changes.
Replaces a regular expression pattern with repl in each "name: value"
header line.
Returns:
The number of replacements made.
"""
repl_count = 0
fields = []
for name, value in headers.fields:
line = name + b": " + value
inner_repl_count = 0
for pattern, replacement in modifiers_dict.items():
line, n = pattern.subn(replacement, line)
inner_repl_count += n
if len(line) == 0:
# No need to go though other patterns for this line
break
if len(line) == 0:
# Skip (remove) this header line in this case
break
if inner_repl_count > 0:
# only in case when there were some replacements:
try:
name, value = line.split(b": ", 1)
except ValueError:
# We get a ValueError if the replacement removed the ": "
# There's not much we can do about this, so we just keep the header as-is.
pass
else:
repl_count += inner_repl_count
fields.append((name, value))
headers.fields = tuple(fields)
return repl_count
def evaluate(self, flow: HTTPFlow, phase: Phase):
flow_phase_obj = getattr(flow, phase.value.lower())
input_message = HttpMessage(
url=flow.request.url,
headers=dict([(name, value)
for name, value in flow_phase_obj.headers.items()]),
data=flow_phase_obj.content.decode(),
)
error = None
output_message = None
with measure_execution_time() as exc_time_ctx:
try:
output_message = evaluate(self.code, input_message)
except Exception as exc:
logger.exception(exc)
error = str(exc)
audit_logs.emit(
OperationLogRecord(
flow_id=flow.id,
proxy_mode=get_proxy_context().mode,
route_id=self._route_id,
filter_id=self._filter_id,
phase=phase,
operation_name=self.operation_name,
execution_time_ms=exc_time_ctx.elapsed_ms,
execution_time_ns=exc_time_ctx.elapsed_ns,
status=(OperationStatus.ERROR
if error else OperationStatus.OK),
error_message=error,
))
if output_message:
if input_message.headers != output_message.headers:
flow_phase_obj.headers = Headers([
(name.encode('UTF-8'), value.encode('UTF-8'))
for name, value in output_message.headers.items()
])
if input_message.data != output_message.data:
flow_phase_obj.text = output_message.data
def test_request_interceptor_creates_response(self):
mock_flow = Mock()
mock_flow.request.url = 'http://somewhere.com/some/path'
mock_flow.request.method = 'GET'
mock_flow.request.headers = Headers([(b'Accept-Encoding', b'identity')
])
mock_flow.request.raw_content = b''
def intercept(req):
req.create_response(status_code=200,
headers={'a': 'b'},
body=b'foobarbaz')
self.server.request_interceptor = intercept
self.handler.request(mock_flow)
self.assertEqual(200, mock_flow.response.status_code)
self.assertEqual({
'a': 'b',
'content-length': '9'
}, dict(mock_flow.response.headers))
self.assertEqual(b'foobarbaz', mock_flow.response.content)
def test_replace_with_count(self):
headers = Headers(Host="foobarfoo.com", Accept="foo/bar")
replacements = headers.replace("foo", "bar", count=1)
assert replacements == 1
def test_replace_remove_spacer(self):
headers = Headers(Host="example.com")
replacements = headers.replace(r"Host: ", "X-Host ")
assert replacements == 0
assert headers["Host"] == "example.com"
TEST_DICT_HEADERS = {
'Content-Type': ['text/plain; charset=ASCII'],
'Access-Control-Expose-Headers': [
'Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, '
'X-Multiple-Values',
],
'X-Multiple-Values': [
'Value1',
'Value2',
]
}
TEST_MITM_HEADERS = Headers(fields=[
(b'Content-Type', b'text/plain; charset=ASCII'),
(b'Access-Control-Expose-Headers',
b'Content-Type, ETag, Link, X-RateLimit-Limit, '
b'X-RateLimit-Remaining, X-RateLimit-Reset, X-Multiple-Values'),
(b'X-Multiple-Values', b'Value1'),
(b'X-Multiple-Values', b'Value2'),
])
TEST_HEADERS = MITMHeaders(
headers={
'Content-Type': ['text/plain; charset=ASCII'],
'Access-Control-Expose-Headers': [
'Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, '
'X-Multiple-Values',
],
'X-Multiple-Values': [
'Value1',
'Value2',
]
def _to_headers_obj(self, headers):
return Headers([(k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()])
def test_replace_remove_spacer(self):
headers = Headers(Host="example.com")
replacements = headers.replace(r"Host: ", "X-Host ")
assert replacements == 0
assert headers["Host"] == "example.com"
def test_replace_with_count(self):
headers = Headers(Host="foobarfoo.com", Accept="foo/bar")
replacements = headers.replace("foo", "bar", count=1)
assert replacements == 1
'body': b'x\x9c\xf3H\xcd\xc9\xc9\xd7Q(\xcf/\xcaIQ\x04\x00 ^\x04\x8a',
'headers': {
'Content-Type': ['text/plain; charset=UTF-8'],
'Date': ['Wed, 21 Mar 2018 12:47:18 GMT'],
'Server': ['nginx/1.12.2'],
'Content-Encoding': ['gzip']
}
}
TEST_MITM_RESPONSE = HTTPResponse(
http_version='HTTP/1.1',
status_code=200,
reason='OK',
headers=Headers(fields=[
(b'Content-Type', b'text/plain; charset=ISO-8859-2'),
(b'Date', b'Wed, 21 Mar 2018 12:47:18 GMT'),
(b'Server', b'nginx/1.12.2'),
]),
content=b'Witaj, \xb6wiecie!',
)
TEST_MITM_RESPONSE_GZIP = HTTPResponse(
http_version='HTTP/1.1',
status_code=200,
reason='OK',
headers=Headers(fields=[
(b'Content-Type', b'text/plain; charset=UTF-8'),
(b'Date', b'Wed, 21 Mar 2018 12:47:18 GMT'),
(b'Server', b'nginx/1.12.2'),
(b'Content-Encoding', b'gzip'),
]),
def from_state(cls, state):
state["headers"] = Headers.from_state(state["headers"])
if state["trailers"] is not None:
state["trailers"] = Headers.from_state(state["trailers"])
return cls(**state)
def set_state(self, state):
for k, v in state.items():
if k in ("headers", "trailers") and v is not None:
v = Headers.from_state(v)
setattr(self, k, v)
def request(flow):
direction = "REQUEST"
q = flow.request.query
flow.original = flow.copy()
original = flow.original
changed_q = False
changed_h = False
changed_host = False
changed_path = False
for k in conf.keys():
conf1 = conf[k]
escape_match = False
try:
escape_match = re.match(re.escape(k), original.request.url)
except Exception as e:
pass
if k == "all" or escape_match or k and re.match(k, original.request.url):
if "request" in conf1 and "path" in conf1["request"]:
path = conf1["request"]["path"]
if "replace" in path:
path = path["replace"]
flow.request.path = realv(path, flow)
changed_path = True
if "request" in conf1 and "protocol" in conf1["request"]:
protocol = conf1["request"]["protocol"]
if "replace" in protocol:
protocol = protocol["replace"]
flow.request.scheme = realv(protocol, flow)
if "request" in conf1 and "port" in conf1["request"]:
port = conf1["request"]["port"]
if "replace" in port:
port = port["replace"]
flow.request.port = realv(port, flow)
if "request" in conf1 and "host" in conf1["request"]:
host = conf1["request"]["host"]
if "replace" in host:
host = host["replace"]
flow.request.host = realv(host, flow)
changed_host = True
if "request" in conf1 and "query" in conf1["request"]:
query = conf1["request"]["query"]
if "replace" in query:
query = query["replace"]
for k,v in query.items():
q[k] = [realv(v, flow)]
changed_q = True
if "request" in conf1 and "redirect" in conf1["request"]:
redirect = conf1["request"]["redirect"]
redirect = realv(redirect, flow)
resp = HTTPResponse(
b"HTTP/1.1", 302, b"OK",
Headers(Location=redirect.encode('utf-8')),
b""
)
flow.reply(resp)
if "request" in conf1 and "header" in conf1["request"]:
headers = conf1["request"]["header"]
changed_h = changed_h or process_headers(flow, headers, direction)
if "request" in conf1 and "body" in conf1["request"]:
body = conf1["request"]["body"]
process_body(flow, body, direction)
if changed_q:
flow.request.set_query(q)
if changed_h:
print(flow.request.headers)
if changed_host:
flow.request.update_host_header()