def test_response_interceptor_called(self):
self.mock_flow.request.id = '12345'
self.mock_flow.request.url = 'http://somewhere.com/some/path'
self.mock_flow.request.headers = Headers()
self.mock_flow.request.raw_content = b''
self.mock_flow.response.status_code = 200
self.mock_flow.response.reason = 'OK'
self.mock_flow.response.headers = Headers([(b'Content-Length', b'6')])
self.mock_flow.response.raw_content = b'foobar'
def intercept(req, res):
if req.url == 'http://somewhere.com/some/path':
res.status_code = 201
res.reason = 'Created'
res.headers['a'] = 'b'
res.body = b'foobarbaz'
self.proxy.response_interceptor = intercept
self.handler.response(self.mock_flow)
self.assertEqual(201, self.mock_flow.response.status_code)
self.assertEqual('Created', self.mock_flow.response.reason)
self.assertEqual({
'Content-Length': '6',
'a': 'b'
}, dict(self.mock_flow.response.headers))
self.assertEqual(b'foobarbaz', self.mock_flow.response.raw_content)
def __init__(
self,
http_version: bytes,
status_code: int,
reason: bytes,
headers: Union[Headers, Tuple[Tuple[bytes, bytes], ...]],
content: Optional[bytes],
trailers: Union[None, Headers, Tuple[Tuple[bytes, bytes], ...]],
timestamp_start: float,
timestamp_end: Optional[float],
):
# auto-convert invalid types to retain compatibility with older code.
if isinstance(http_version, str):
http_version = http_version.encode("ascii", "strict")
if isinstance(reason, str):
reason = reason.encode("ascii", "strict")
if isinstance(content, str):
raise ValueError("Content must be bytes, not {}".format(type(content).__name__))
if not isinstance(headers, Headers):
headers = Headers(headers)
if trailers is not None and not isinstance(trailers, Headers):
trailers = Headers(trailers)
self.data = ResponseData(
http_version=http_version,
status_code=status_code,
reason=reason,
headers=headers,
content=content,
trailers=trailers,
timestamp_start=timestamp_start,
timestamp_end=timestamp_end,
)
def make(
cls,
method: str,
url: str,
content: Union[bytes, str] = "",
headers: Union[Headers, Dict[Union[str, bytes], Union[str, bytes]], Iterable[Tuple[bytes, bytes]]] = ()
) -> "Request":
"""
Simplified API for creating request objects.
"""
# Headers can be list or dict, we differentiate here.
if isinstance(headers, Headers):
pass
elif isinstance(headers, dict):
headers = Headers(
(always_bytes(k, "utf-8", "surrogateescape"),
always_bytes(v, "utf-8", "surrogateescape"))
for k, v in headers.items()
)
elif isinstance(headers, Iterable):
headers = Headers(headers)
else:
raise TypeError("Expected headers to be an iterable or dict, but is {}.".format(
type(headers).__name__
))
req = cls(
"",
0,
method.encode("utf-8", "surrogateescape"),
b"",
b"",
b"",
b"HTTP/1.1",
headers,
b"",
None,
time.time(),
time.time(),
)
req.url = url
# Assign this manually to update the content-length header.
if isinstance(content, bytes):
req.content = content
elif isinstance(content, str):
req.text = content
else:
raise TypeError(f"Expected content to be str or bytes, but is {type(content).__name__}.")
return req
def __init__(
self,
host: str,
port: int,
method: bytes,
scheme: bytes,
authority: bytes,
path: bytes,
http_version: bytes,
headers: Union[Headers, Tuple[Tuple[bytes, bytes], ...]],
content: Optional[bytes],
trailers: Union[None, Headers, Tuple[Tuple[bytes, bytes], ...]],
timestamp_start: float,
timestamp_end: Optional[float],
):
# auto-convert invalid types to retain compatibility with older code.
if isinstance(host, bytes):
host = host.decode("idna", "strict")
if isinstance(method, str):
method = method.encode("ascii", "strict")
if isinstance(scheme, str):
scheme = scheme.encode("ascii", "strict")
if isinstance(authority, str):
authority = authority.encode("ascii", "strict")
if isinstance(path, str):
path = path.encode("ascii", "strict")
if isinstance(http_version, str):
http_version = http_version.encode("ascii", "strict")
if isinstance(content, str):
raise ValueError(f"Content must be bytes, not {type(content).__name__}")
if not isinstance(headers, Headers):
headers = Headers(headers)
if trailers is not None and not isinstance(trailers, Headers):
trailers = Headers(trailers)
self.data = RequestData(
host=host,
port=port,
method=method,
scheme=scheme,
authority=authority,
path=path,
http_version=http_version,
headers=headers,
content=content,
trailers=trailers,
timestamp_start=timestamp_start,
timestamp_end=timestamp_end,
)
def test_request_interceptor_called(self):
self.mock_flow.request.url = 'http://somewhere.com/some/path'
self.mock_flow.request.method = 'GET'
self.mock_flow.request.headers = Headers([(b'Accept-Encoding',
b'identity')])
self.mock_flow.request.raw_content = b''
def intercept(req):
req.method = 'POST'
req.url = 'https://www.google.com/foo/bar?x=y'
req.body = b'foobarbaz'
req.headers['a'] = 'b'
self.proxy.request_interceptor = intercept
self.handler.request(self.mock_flow)
self.assertEqual('POST', self.mock_flow.request.method)
self.assertEqual('https://www.google.com/foo/bar?x=y',
self.mock_flow.request.url)
self.assertEqual({
'Accept-Encoding': 'identity',
'a': 'b'
}, dict(self.mock_flow.request.headers))
self.assertEqual(b'foobarbaz', self.mock_flow.request.raw_content)
def test_save_request(self):
self.mock_flow.request.url = 'http://somewhere.com/some/path'
self.mock_flow.request.method = 'GET'
self.mock_flow.request.headers = Headers([(b'Accept-Encoding',
b'identity')])
self.mock_flow.request.raw_content = b'foobar'
saved_request = None
def save_request(req):
nonlocal saved_request
req.id = '12345'
saved_request = req
self.proxy.storage.save_request.side_effect = save_request
self.handler.request(self.mock_flow)
self.assertEqual(1, self.proxy.storage.save_request.call_count)
self.assertEqual('GET', saved_request.method)
self.assertEqual('http://somewhere.com/some/path', saved_request.url)
self.assertEqual({'Accept-Encoding': 'identity'},
dict(saved_request.headers))
self.assertEqual(b'foobar', saved_request.body)
self.assertEqual('12345', saved_request.id)
self.assertEqual('12345', self.mock_flow.request.id)
def make(
cls,
status_code: int = 200,
content: Union[bytes, str] = b"",
headers: Union[Headers, Mapping[str, Union[str, bytes]], Iterable[Tuple[bytes, bytes]]] = ()
) -> "Response":
"""
Simplified API for creating response objects.
"""
if isinstance(headers, Headers):
headers = headers
elif isinstance(headers, dict):
headers = Headers(
(always_bytes(k, "utf-8", "surrogateescape"),
always_bytes(v, "utf-8", "surrogateescape"))
for k, v in headers.items()
)
elif isinstance(headers, Iterable):
headers = Headers(headers)
else:
raise TypeError("Expected headers to be an iterable or dict, but is {}.".format(
type(headers).__name__
))
resp = cls(
b"HTTP/1.1",
status_code,
status_codes.RESPONSES.get(status_code, "").encode(),
headers,
None,
None,
time.time(),
time.time(),
)
# Assign this manually to update the content-length header.
if isinstance(content, bytes):
resp.content = content
elif isinstance(content, str):
resp.text = content
else:
raise TypeError(f"Expected content to be str or bytes, but is {type(content).__name__}.")
return resp
def test_request_modifier_called(self):
self.mock_flow.request.url = 'http://somewhere.com/some/path'
self.mock_flow.request.method = 'GET'
self.mock_flow.request.headers = Headers([(b'Accept-Encoding',
b'identity')])
self.mock_flow.request.raw_content = b''
self.handler.request(self.mock_flow)
self.proxy.modifier.modify_request.assert_called_once_with(
self.mock_flow.request, bodyattr='raw_content')
def test_save_har_entry_disabled(self, mock_har):
self.proxy.options['enable_har'] = False
self.mock_flow.request.id = '12345'
self.mock_flow.response.headers = Headers()
self.mock_flow.response.raw_content = b''
mock_har.create_har_entry.return_value = {'name': 'test_har_entry'}
self.handler.response(self.mock_flow)
self.proxy.storage.save_har_entry.assert_not_called()
mock_har.create_har_entry.assert_not_called()
def test_save_response(self):
self.mock_flow.request.id = '12345'
self.mock_flow.request.url = 'http://somewhere.com/some/path'
self.mock_flow.response.status_code = 200
self.mock_flow.response.reason = 'OK'
self.mock_flow.response.headers = Headers([(b'Content-Length', b'6')])
self.mock_flow.response.raw_content = b'foobar'
mock_cert = Mock()
mock_cert.subject = [(b'C', b'US'), (b'O', b'Mozilla Corporation')]
mock_cert.serial = 123456789
mock_cert.keyinfo = ('RSA', 2048)
mock_cert.x509.get_signature_algorithm.return_value = b'test_algo'
mock_cert.has_expired = False
mock_cert.issuer = [(b'CN', b'DigiCert SHA2 Secure Server CA')]
mock_cert.organization = b'Mozilla Corporation'
mock_cert.cn = b'*.cdn.mozilla.net'
mock_cert.altnames = [b'*.cdn.mozilla.net', b'cdn.mozilla.net']
notbefore = datetime.now()
notafter = datetime.now()
mock_cert.notbefore = notbefore
mock_cert.notafter = notafter
self.mock_flow.server_conn.cert = mock_cert
saved_response = None
def save_response(_, response):
nonlocal saved_response
saved_response = response
self.proxy.storage.save_response.side_effect = save_response
self.handler.response(self.mock_flow)
self.proxy.storage.save_response.assert_called_once_with(
'12345', saved_response)
self.assertEqual(200, saved_response.status_code)
self.assertEqual('OK', saved_response.reason)
self.assertEqual({'Content-Length': '6'}, dict(saved_response.headers))
self.assertEqual(b'foobar', saved_response.body)
self.assertEqual([(b'C', b'US'), (b'O', b'Mozilla Corporation')],
saved_response.cert['subject'])
self.assertEqual(123456789, saved_response.cert['serial'])
self.assertEqual(('RSA', 2048), saved_response.cert['key'])
self.assertEqual(b'test_algo',
saved_response.cert['signature_algorithm'])
self.assertFalse(saved_response.cert['expired'])
self.assertEqual([(b'CN', b'DigiCert SHA2 Secure Server CA')],
saved_response.cert['issuer'])
self.assertEqual(notbefore, saved_response.cert['notbefore'])
self.assertEqual(notafter, saved_response.cert['notafter'])
self.assertEqual(b'Mozilla Corporation',
saved_response.cert['organization'])
self.assertEqual(b'*.cdn.mozilla.net', saved_response.cert['cn'])
self.assertEqual([b'*.cdn.mozilla.net', b'cdn.mozilla.net'],
saved_response.cert['altnames'])
def test_ignore_request_method_out_of_scope(self):
self.mock_flow.request.url = 'https://server2/some/path'
self.mock_flow.request.method = 'GET'
self.mock_flow.request.headers = Headers([(b'Accept-Encoding',
b'identity')])
self.mock_flow.request.raw_content = b'foobar'
self.proxy.options['ignore_http_methods'] = ['GET']
self.handler.request(self.mock_flow)
self.proxy.storage.save_request.assert_not_called()
def test_disable_encoding(self):
self.mock_flow.request.url = 'http://somewhere.com/some/path'
self.mock_flow.request.method = 'GET'
self.mock_flow.request.headers = Headers([(b'Accept-Encoding', b'gzip')
])
self.mock_flow.request.raw_content = b''
self.proxy.options['disable_encoding'] = True
self.handler.request(self.mock_flow)
self.assertEqual({'Accept-Encoding': 'identity'},
dict(self.mock_flow.request.headers))
def test_response_modifier_called(self):
self.mock_flow.request.id = '12345'
self.mock_flow.request.url = 'http://somewhere.com/some/path'
self.mock_flow.response.status_code = 200
self.mock_flow.response.reason = 'OK'
self.mock_flow.response.headers = Headers([(b'Content-Length', b'6')])
self.mock_flow.response.raw_content = b'foobar'
self.handler.response(self.mock_flow)
self.proxy.modifier.modify_response.assert_called_once_with(
self.mock_flow.response, self.mock_flow.request)
def test_ignore_request_url_out_of_scope(self):
mock_flow = Mock()
mock_flow.request.url = 'https://server2/some/path'
mock_flow.request.method = 'GET'
mock_flow.request.headers = Headers([(b'Accept-Encoding', b'identity')
])
mock_flow.request.raw_content = b'foobar'
self.proxy.scopes = ['https://server1.*']
# self.handler.requestheaders(mock_flow)
self.handler.request(mock_flow)
self.assertEqual(0, self.proxy.storage.save_request.call_count)
def test_multiple_response_headers(self):
self.mock_flow.request.id = '12345'
self.mock_flow.request.url = 'http://somewhere.com/some/path'
self.mock_flow.response.status_code = 200
self.mock_flow.response.reason = 'OK'
self.mock_flow.response.headers = Headers([(b'Set-Cookie', b'12345'),
(b'Set-Cookie', b'67890')])
self.mock_flow.response.raw_content = b'foobar'
saved_response = None
def save_response(_, response):
nonlocal saved_response
saved_response = response
self.proxy.storage.save_response.side_effect = save_response
self.handler.response(self.mock_flow)
self.assertEqual([('Set-Cookie', '12345'), ('Set-Cookie', '67890')],
saved_response.headers.items())
def test_save_response(self):
mock_flow = Mock()
mock_flow.request.id = '12345'
mock_flow.request.url = 'http://somewhere.com/some/path'
mock_flow.response.status_code = 200
mock_flow.response.reason = 'OK'
mock_flow.response.headers = Headers([(b'Content-Length', b'6')])
mock_flow.response.raw_content = b'foobar'
mock_cert = Mock()
mock_cert.subject = 'test_subject'
mock_cert.serial = 'test_serial'
mock_cert.keyinfo = 'test_key'
mock_cert.x509.get_signature_algorithm.return_value = 'test_algo'
mock_cert.has_expired = False
mock_cert.issuer = 'test_issuer'
mock_flow.server_conn.cert = mock_cert
saved_response = None
def save_response(_, response):
nonlocal saved_response
saved_response = response
self.proxy.storage.save_response.side_effect = save_response
self.handler.response(mock_flow)
self.proxy.storage.save_response.assert_called_once_with(
'12345', saved_response)
self.assertEqual(200, saved_response.status_code)
self.assertEqual('OK', saved_response.reason)
self.assertEqual({'Content-Length': '6'}, dict(saved_response.headers))
self.assertEqual(b'foobar', saved_response.body)
self.assertEqual('test_subject', saved_response.cert['subject'])
self.assertEqual('test_serial', saved_response.cert['serial'])
self.assertEqual('test_key', saved_response.cert['key'])
self.assertEqual('test_algo',
saved_response.cert['signature_algorithm'])
self.assertFalse(saved_response.cert['expired'])
self.assertEqual('test_issuer', saved_response.cert['issuer'])
def test_request_interceptor_creates_response(self):
self.mock_flow.request.url = 'http://somewhere.com/some/path'
self.mock_flow.request.method = 'GET'
self.mock_flow.request.headers = Headers([(b'Accept-Encoding',
b'identity')])
self.mock_flow.request.raw_content = b''
def intercept(req):
req.create_response(status_code=200,
headers={'a': 'b'},
body=b'foobarbaz')
self.proxy.request_interceptor = intercept
self.handler.request(self.mock_flow)
self.assertEqual(200, self.mock_flow.response.status_code)
self.assertEqual({
'a': 'b',
'content-length': '9'
}, dict(self.mock_flow.response.headers))
self.assertEqual(b'foobarbaz', self.mock_flow.response.content)
def test_save_response(self):
mock_flow = Mock()
mock_flow.request.id = '12345'
mock_flow.request.url = 'http://somewhere.com/some/path'
mock_flow.response.status_code = 200
mock_flow.response.reason = 'OK'
mock_flow.response.headers = Headers([(b'Content-Length', b'6')])
mock_flow.response.raw_content = b'foobar'
saved_response = None
def save_response(_, response):
nonlocal saved_response
saved_response = response
self.proxy.storage.save_response.side_effect = save_response
self.handler.response(mock_flow)
self.proxy.storage.save_response.assert_called_once_with(
'12345', saved_response)
self.assertEqual(200, saved_response.status_code)
self.assertEqual('OK', saved_response.reason)
self.assertEqual({'Content-Length': '6'}, dict(saved_response.headers))
self.assertEqual(b'foobar', saved_response.body)
def set_state(self, state):
for k, v in state.items():
if k in ("headers", "trailers") and v is not None:
v = Headers.from_state(v)
setattr(self, k, v)
def from_state(cls, state):
state["headers"] = Headers.from_state(state["headers"])
if state["trailers"] is not None:
state["trailers"] = Headers.from_state(state["trailers"])
return cls(**state)
def _to_headers_obj(self, headers):
return Headers([(k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()])
