def __init__(self, host, port, options): self.options = options # Used to stored captured requests self.storage = RequestStorage( base_dir=options.pop('request_storage_base_dir', None)) extract_cert_and_key(self.storage.home_dir) # Used to modify requests/responses passing through the server # DEPRECATED. Will be superceded by request/response interceptors. self.modifier = RequestModifier() # The scope of requests we're interested in capturing. self.scopes = [] self.request_interceptor = None self.response_interceptor = None self._event_loop = asyncio.new_event_loop() # mitmproxy specific options mitmproxy_opts = Options( confdir=self.storage.home_dir, listen_host=host, listen_port=port, ) # Create an instance of the mitmproxy server self._master = Master(self._event_loop, mitmproxy_opts) self._master.server = ProxyServer(ProxyConfig(mitmproxy_opts)) self._master.addons.add(*addons.default_addons()) self._master.addons.add(SendToLogger()) self._master.addons.add(InterceptRequestHandler(self)) # Update the options now all addons have been added mitmproxy_opts.update( ssl_insecure=options.get('verify_ssl', True), upstream_cert=DEFAULT_UPSTREAM_CERT, stream_websockets=DEFAULT_STREAM_WEBSOCKETS, suppress_connection_errors=options.get( 'suppress_connection_errors', True), **self._get_upstream_proxy_args(), ) if options.get('disable_capture', False): self.scopes = ['$^'] mitmproxy_opts.update(ignore_hosts=['.*']) # Options that are prefixed mitm_ are passed through to mitmproxy mitmproxy_opts.update( **{k[5:]: v for k, v in options.items() if k.startswith('mitm_')})
def setUp(self): self.modifier = RequestModifier()
class RequestModifierTest(TestCase): def setUp(self): self.modifier = RequestModifier() def test_override_header(self): self.modifier.headers = {'User-Agent': 'Test_User_Agent_String'} mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual('Test_User_Agent_String', mock_request.headers['User-Agent']) def test_override_header_ignore_response_header(self): self.modifier.headers = { 'User-Agent': 'Test_User_Agent_String', 'response:Cache-Control': 'none' } mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual('Test_User_Agent_String', mock_request.headers['User-Agent']) self.assertNotIn('response:Cache-Control', mock_request.headers) def test_override_header_with_single_url_matching(self): self.modifier.headers = [(".*prod1.server.com.*", { 'User-Agent': 'Test_User_Agent_String' })] mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual('Test_User_Agent_String', mock_request.headers['User-Agent']) def test_override_multiple_headers_with_single_url_matching(self): self.modifier.headers = [(".*prod1.server.com.*", { 'User-Agent': 'Test_User_Agent_String', 'New-Header': 'HeaderValue' })] mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual('Test_User_Agent_String', mock_request.headers['User-Agent']) self.assertEqual('HeaderValue', mock_request.headers['New-Header']) def test_override_headers_with_multiple_url_matching(self): self.modifier.headers = [(".*prod1.server.com.*", { 'User-Agent': 'Test_User_Agent_String', 'New-Header': 'HeaderValue' }), (".*prod2.server.com.*", { 'User-Agent2': 'Test_User_Agent_String2', 'New-Header2': 'HeaderValue' })] url = "https://prod1.server.com/some/path/12345" mock_request = self._create_mock_request(url) self.modifier.modify_request(mock_request) self.assertEqual('Test_User_Agent_String', mock_request.headers['User-Agent']) self.assertEqual(url, mock_request.url) self.assertFalse('User-Agent2' in mock_request.headers or 'New-Header2' in mock_request.headers) url = "https://prod2.server.com/some/path/12345" mock_request = self._create_mock_request(url) self.modifier.modify_request(mock_request) self.assertEqual('HeaderValue', mock_request.headers['New-Header2']) self.assertEqual(url, mock_request.url) self.assertFalse('New-Header' in mock_request.headers) def test_override_header_with_url_not_matching(self): self.modifier.headers = [(".*prod.server.com.*", { 'User-Agent': 'Test_User_Agent_String' })] mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual( 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 ' 'Firefox/10.0', mock_request.headers['User-Agent'], ) def test_override_header_case_insensitive(self): self.modifier.headers = {'user-agent': 'Test_User_Agent_String'} mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual('Test_User_Agent_String', mock_request.headers['User-Agent']) def test_add_new_header(self): self.modifier.headers = {'New-Header': 'Some-Value'} mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual('Some-Value', mock_request.headers['New-Header']) def test_filter_out_header(self): self.modifier.headers = {'User-Agent': None} mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertNotIn('User-Agent', mock_request.headers) def test_filter_out_non_existent_header(self): self.modifier.headers = { 'Host': None # Does not exist in the request } mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertNotIn('Host', mock_request.headers) def test_override_response_header(self): self.modifier.headers = { 'User-Agent': 'Test_User_Agent_String', 'response:Cache-Control': 'max-age=2592000' } mock_request = self._create_mock_request() mock_response = self._create_mock_response() self.modifier.modify_response(mock_response, mock_request) self.assertEqual(1, len(mock_response.headers)) self.assertEqual('max-age=2592000', mock_response.headers['Cache-Control']) def test_override_response_header_with_url_not_matching(self): self.modifier.headers = [(".*prod.server.com.*", { 'response:Cache-Control': 'max-age=2592000' })] mock_request = self._create_mock_request() mock_response = self._create_mock_response() self.modifier.modify_response(mock_response, mock_request) self.assertEqual('none', mock_response.headers['Cache-Control']) def test_override_response_header_case_insensitive(self): self.modifier.headers = { 'User-Agent': 'Test_User_Agent_String', 'respoNse:cache-control': 'max-age=2592000' } mock_request = self._create_mock_request() mock_response = self._create_mock_response() self.modifier.modify_response(mock_response, mock_request) self.assertEqual(1, len(mock_response.headers)) self.assertEqual('max-age=2592000', mock_response.headers['Cache-Control']) def test_add_new_response_header(self): self.modifier.headers = { 'User-Agent': 'Test_User_Agent_String', 'response:New-Header': 'Some-Value' } mock_request = self._create_mock_request() mock_response = self._create_mock_response() self.modifier.modify_response(mock_response, mock_request) self.assertEqual(2, len(mock_response.headers)) self.assertEqual('Some-Value', mock_response.headers['New-Header']) def test_filter_out_response_header(self): self.modifier.headers = { 'User-Agent': 'Test_User_Agent_String', 'response:Cache-Control': None } mock_request = self._create_mock_request() mock_response = self._create_mock_response() self.modifier.modify_response(mock_response, mock_request) self.assertNotIn('Cache-Control', mock_response.headers) def test_filter_out_non_existent_response_header(self): self.modifier.headers = { 'response:Host': None # Does not exist in the response } mock_request = self._create_mock_request() mock_response = self._create_mock_response() self.modifier.modify_response(mock_response, mock_request) self.assertNotIn('Host', mock_response.headers) def test_clear_header_overrides(self): self.modifier.headers = {'User-Agent': 'Test_User_Agent_String'} mock_request = self._create_mock_request() del self.modifier.headers self.modifier.modify_request(mock_request) self.assertEqual( 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/' '20100101 Firefox/10.0', mock_request.headers['User-Agent']) def test_get_header_overrides(self): self.modifier.headers = {'User-Agent': 'Test_User_Agent_String'} self.assertEqual({'User-Agent': 'Test_User_Agent_String'}, self.modifier.headers) def test_override_param_qs(self): self.modifier.params = {'foo': 'baz'} mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual({ 'foo': 'baz', 'spam': 'eggs' }, self._extract_params(mock_request.url)) def test_override_param_body(self): self.modifier.params = {'foo': 'bazz'} mock_request = self._create_mock_request( url='https://prod1.server.com/some/path/12345', method='POST', headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=b'foo=bar&spam=eggs') self.modifier.modify_request(mock_request) qs = parse_qs(mock_request.body.decode('utf-8')) self.assertEqual(2, len(qs)) self.assertEqual('bazz', qs['foo'][0]) self.assertEqual('eggs', qs['spam'][0]) self.assertEqual('18', mock_request.headers['Content-Length']) def test_override_multiple_params(self): self.modifier.params = {'foo': 'baz', 'spam': 'ham'} mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual({ 'foo': 'baz', 'spam': 'ham' }, self._extract_params(mock_request.url)) def test_override_param_multiple_values(self): self.modifier.params = {'foo': ['a', 'b']} mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) params = self._extract_params(mock_request.url, flatten=False) self.assertEqual(['a', 'b'], params['foo']) self.assertEqual(['eggs'], params['spam']) def test_override_param_body_no_content_type(self): self.modifier.params = {'foo': 'baz'} mock_request = self._create_mock_request( url='https://prod1.server.com/some/path/12345?foo=baz&spam=eggs', method='POST', body=b'foo=bar&spam=eggs') self.modifier.modify_request(mock_request) self.assertEqual(b'foo=bar&spam=eggs', mock_request.body) def test_override_param_with_single_url_matching(self): self.modifier.params = [(".*prod1.server.com.*", {'foo': 'baz'})] mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual({ 'foo': 'baz', 'spam': 'eggs' }, self._extract_params(mock_request.url)) def test_override_multiple_params_with_single_url_matching(self): self.modifier.params = [(".*prod1.server.com.*", { 'foo': 'baz', 'spam': 'ham' })] mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual({ 'foo': 'baz', 'spam': 'ham' }, self._extract_params(mock_request.url)) def test_override_params_with_multiple_url_matching(self): self.modifier.params = [(".*prod1.server.com.*", { 'foo': 'baz', 'spam': 'ham' }), (".*prod2.server.com.*", { 'foo': 'baz2', 'spam': 'ham2' })] # Modify a request that matches the first pattern url = 'https://prod1.server.com/some/path/12345?foo=bar&spam=eggs' mock_request = self._create_mock_request(url) self.modifier.modify_request(mock_request) self.assertEqual({ 'foo': 'baz', 'spam': 'ham' }, self._extract_params(mock_request.url)) # Modify a request that matches the second pattern url = 'https://prod2.server.com/some/path/12345?foo=bar&spam=eggs' mock_request = self._create_mock_request(url) self.modifier.modify_request(mock_request) self.assertEqual({ 'foo': 'baz2', 'spam': 'ham2' }, self._extract_params(mock_request.url)) def test_override_param_with_url_not_matching(self): self.modifier.params = [(".*prod.server.com.*", {'foo': 'baz'})] mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual({ 'foo': 'bar', 'spam': 'eggs' }, self._extract_params(mock_request.url)) def test_add_new_param_qs(self): self.modifier.params = {'foo': 'baz'} mock_request = self._create_mock_request( 'https://prod1.server.com/some/path/12345?spam=eggs') self.modifier.modify_request(mock_request) self.assertEqual({ 'foo': 'baz', 'spam': 'eggs' }, self._extract_params(mock_request.url)) def test_add_new_param_body(self): self.modifier.params = {'foo': 'bazz'} mock_request = self._create_mock_request( url='https://prod1.server.com/some/path/12345', method='POST', headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=b'spam=eggs') self.modifier.modify_request(mock_request) qs = parse_qs(mock_request.body.decode('utf-8')) self.assertEqual(2, len(qs)) self.assertEqual('bazz', qs['foo'][0]) self.assertEqual('eggs', qs['spam'][0]) self.assertEqual('18', mock_request.headers['Content-Length']) def test_add_param_no_qs(self): self.modifier.params = {'foo': 'baz'} mock_request = self._create_mock_request( 'https://prod1.server.com/some/path/12345') self.modifier.modify_request(mock_request) self.assertEqual({ 'foo': 'baz', }, self._extract_params(mock_request.url)) def test_add_param_no_body(self): self.modifier.params = {'foo': 'bazz'} mock_request = self._create_mock_request( url='https://prod1.server.com/some/path/12345', method='POST', headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=b'') self.modifier.modify_request(mock_request) self.assertEqual(b'foo=bazz', mock_request.body) self.assertEqual('8', mock_request.headers['Content-Length']) def test_override_bodies_with_url_not_matching(self): self.modifier.params = [(".*prod.server.com.*", {'foo': 'baz'})] mock_request = self._create_mock_request(method='POST') self.modifier.modify_request(mock_request) self.assertEqual(b'', mock_request.body) def test_filter_out_param_qs(self): self.modifier.params = {'foo': None} mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual({'spam': 'eggs'}, self._extract_params(mock_request.url)) def test_filter_out_param_body(self): self.modifier.params = {'foo': None} mock_request = self._create_mock_request( url='https://prod1.server.com/some/path/12345', method='POST', headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=b'foo=bar&spam=eggs') self.modifier.modify_request(mock_request) self.assertEqual(b'spam=eggs', mock_request.body) self.assertEqual('9', mock_request.headers['Content-Length']) def test_filter_out_non_existent_param(self): self.modifier.params = { 'hello': None # Does not exist in the request } mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual({ 'foo': 'bar', 'spam': 'eggs' }, self._extract_params(mock_request.url)) def test_clear_param_overrides(self): self.modifier.params = {'foo': 'baz', 'spam': 'ham'} mock_request = self._create_mock_request() del self.modifier.params self.modifier.modify_request(mock_request) self.assertEqual({ 'foo': 'bar', 'spam': 'eggs' }, self._extract_params(mock_request.url)) def test_get_param_overrides(self): self.modifier.params = {'foo': 'baz', 'spam': 'ham'} self.assertEqual({'foo': 'baz', 'spam': 'ham'}, self.modifier.params) def test_override_querystring(self): self.modifier.querystring = 'foo=baz' mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual('https://prod1.server.com/some/path/12345?foo=baz', mock_request.url) def test_override_querystring_with_single_url_matching(self): self.modifier.querystring = [(".*prod1.server.com.*", 'foo=baz')] mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual('https://prod1.server.com/some/path/12345?foo=baz', mock_request.url) def test_override_querystring_with_multiple_url_matching(self): self.modifier.querystring = [(".*prod1.server.com.*", 'foo=baz'), (".*prod2.server.com.*", 'spam=ham')] # Modify a request that matches the first pattern url = 'https://prod1.server.com/some/path/12345?foo=bar&spam=eggs' mock_request = self._create_mock_request(url) self.modifier.modify_request(mock_request) self.assertEqual('https://prod1.server.com/some/path/12345?foo=baz', mock_request.url) # Modify a request that matches the second pattern url = 'https://prod2.server.com/some/path/12345?foo=bar&spam=eggs' mock_request = self._create_mock_request(url) self.modifier.modify_request(mock_request) self.assertEqual('https://prod2.server.com/some/path/12345?spam=ham', mock_request.url) def test_override_querystring_with_url_not_matching(self): self.modifier.querystring = [(".*prod.server.com.*", 'foo=baz')] mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual( 'https://prod1.server.com/some/path/12345?foo=bar&spam=eggs', mock_request.url) def test_add_new_querystring(self): self.modifier.querystring = 'foo=baz' mock_request = self._create_mock_request( 'https://prod1.server.com/some/path/12345') self.modifier.modify_request(mock_request) self.assertEqual('https://prod1.server.com/some/path/12345?foo=baz', mock_request.url) def test_remove_querystring(self): self.modifier.querystring = '' mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual('https://prod1.server.com/some/path/12345', mock_request.url) def test_clear_querystring_overrides(self): self.modifier.querystring = 'foo=baz' mock_request = self._create_mock_request() del self.modifier.querystring self.modifier.modify_request(mock_request) self.assertEqual( 'https://prod1.server.com/some/path/12345?foo=bar&spam=eggs', mock_request.url) def test_rewrite_url(self): self.modifier.rewrite_rules = [ (r'(https?://)prod1.server.com(.*)(\?.*)', r'\1prod2.server.com\2/foo/\3'), ] mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual( 'https://prod2.server.com/some/path/12345/foo/?foo=bar&spam=eggs', mock_request.url) def test_rewrite_url_first_match(self): self.modifier.rewrite_rules = [ (r'(https?://)prod1.server.com(.*)(\?.*)', r'\1prod2.server.com\2/foo/\3'), (r'(https?://)prod1.server.com(.*)(\?.*)', r'\1prod2.server.com\2/bar/\3'), ] mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertEqual( 'https://prod2.server.com/some/path/12345/foo/?foo=bar&spam=eggs', mock_request.url) def test_does_not_rewrite_url(self): self.modifier.rewrite_rules = [ (r'(https?://)prod1.server.com(.*)(\?.*)', r'\1prod2.server.com\2/foo/\3'), ] mock_request = self._create_mock_request() mock_request.url = 'https://prod3.server.com/some/path/12345?foo=bar&spam=eggs' self.modifier.modify_request(mock_request) self.assertEqual( 'https://prod3.server.com/some/path/12345?foo=bar&spam=eggs', mock_request.url) def test_rewrite_url_updates_host_header(self): self.modifier.rewrite_rules = [ (r'(https?://)prod1.server.com(.*)(\?.*)', r'\1prod2.server.com\2/foo/\3'), ] mock_request = self._create_mock_request() mock_request.headers['Host'] = 'prod1.server.com' self.modifier.modify_request(mock_request) self.assertEqual('prod2.server.com', mock_request.headers['Host']) def test_rewrite_url_does_not_update_host_header(self): """Should not update the Host header if it does not already exist.""" self.modifier.rewrite_rules = [ (r'(https?://)prod1.server.com(.*)(\?.*)', r'\1prod2.server.com\2/foo/\3'), ] mock_request = self._create_mock_request() self.modifier.modify_request(mock_request) self.assertNotIn('Host', mock_request.headers) def _create_mock_request( self, url="https://prod1.server.com/some/path/12345?foo=bar&spam=eggs", method='GET', headers=None, body=b''): if headers is None: headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0' } mock_request = Mock() mock_request.url = url mock_request.method = method mock_request.body = body mock_request.headers = headers return mock_request def _create_mock_response(self, headers=None): if headers is None: headers = {'Cache-Control': 'none'} mock_response = Mock() mock_response.headers = headers return mock_response def _extract_params(self, url, flatten=True): return { k: v[0] if flatten else v for k, v in parse_qs(urlsplit(url).query).items() }