def test_cleanup_removes_storage(self):
storage = RequestStorage(base_dir=self.base_dir)
storage._cleanup()
# The 'seleniumwire' parent folder should have been cleaned up
# when there is nothing left inside of it.
self.assertFalse(os.listdir(self.base_dir))
def test_load_request_body(self):
mock_request = self._create_mock_request()
storage = RequestStorage(base_dir=self.base_dir)
request_id = storage.save_request(mock_request, request_body=b'test request body')
request_body = storage.load_request_body(request_id)
self.assertEqual(request_body, b'test request body')
def test_get_cert_dir(self):
storage = RequestStorage(base_dir=self.base_dir)
self.assertTrue(
fnmatch(
storage.get_cert_dir(),
os.path.join(self.base_dir, '.seleniumwire', 'storage-*',
'certs')))
def test_cleanup_does_not_remove_parent_folder(self):
# There is an existing storage folder
os.makedirs(os.path.join(self.base_dir, '.seleniumwire', 'teststorage'))
storage = RequestStorage(base_dir=self.base_dir)
storage.cleanup()
# The existing storage folder is not cleaned up
self.assertEqual(len(os.listdir(self.base_dir)), 1)
self.assertTrue(os.path.exists(os.path.join(self.base_dir, '.seleniumwire', 'teststorage')))
def test_load_request_body(self):
body = b'test request body'
request = self._create_request(body=body)
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request)
request_body = storage.load_request_body(request.id)
self.assertEqual(body, request_body)
def test_load_last_request(self):
request_1 = self._create_request()
request_2 = self._create_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request_1)
storage.save_request(request_2)
last_request = storage.load_last_request()
self.assertEqual(request_2.id, last_request.id)
def __init__(self, host, port, options):
self.options = options
# Used to stored captured requests
self.storage = RequestStorage(
base_dir=options.pop('request_storage_base_dir', None))
# Used to modify requests/responses passing through the server
# Deprecated. Will be superceded by request/response interceptors.
self.modifier = RequestModifier()
# The scope of requests we're interested in capturing.
self.scopes = []
self.request_interceptor = None
self.response_interceptor = None
try:
self._event_loop = asyncio.get_event_loop()
if self._event_loop.is_closed():
# The event loop may be closed if the server had previously
# been shutdown and then spun up again
self._event_loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._event_loop)
except:
self._event_loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._event_loop)
# mitmproxy specific options
mitmproxy_opts = Options(
listen_host=host,
listen_port=port,
)
# Create an instance of the mitmproxy server
self._master = Master(mitmproxy_opts)
self._master.server = ProxyServer(ProxyConfig(mitmproxy_opts))
self._master.addons.add(*addons.default_addons())
self._master.addons.add(SendToLogger())
self._master.addons.add(MitmProxyRequestHandler(self))
# Update the options now all addons have been added
mitmproxy_opts.update(
confdir=DEFAULT_CONFDIR,
ssl_insecure=options.get('verify_ssl', True),
upstream_cert=DEFAULT_UPSTREAM_CERT,
stream_websockets=DEFAULT_STREAM_WEBSOCKETS,
**self._get_upstream_proxy_args(),
)
# Options that are prefixed mitm_ are passed through to mitmproxy
mitmproxy_opts.update(
**{k[5:]: v
for k, v in options.items() if k.startswith('mitm_')})
def test_save_request_with_body(self):
body = b'test request body'
request = self._create_request(body=body)
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request)
request_file_path = self._get_stored_path(request.id, 'request')
with open(request_file_path[0], 'rb') as loaded:
loaded_request = pickle.load(loaded)
self.assertEqual(body, loaded_request.body)
def test_save_request_with_body(self):
mock_request = self._create_mock_request()
request_body = b'test request body'
storage = RequestStorage(base_dir=self.base_dir)
request_id = storage.save_request(mock_request, request_body=request_body)
request_body_path = self._get_stored_path(request_id, 'requestbody')
with open(request_body_path[0], 'rb') as loaded:
loaded_body = pickle.load(loaded)
self.assertEqual(loaded_body, b'test request body')
def test_load_requests(self):
mock_request_1 = self._create_mock_request()
mock_request_2 = self._create_mock_request()
storage = RequestStorage(base_dir=self.base_dir)
request_id1 = storage.save_request(mock_request_1)
request_id2 = storage.save_request(mock_request_2)
requests = storage.load_requests()
self.assertEqual(len(requests), 2)
self.assertEqual(requests[0]['id'], request_id1)
self.assertEqual(requests[1]['id'], request_id2)
self.assertIsNone(requests[0]['response'])
self.assertIsNone(requests[1]['response'])
def test_load_requests(self):
request_1 = self._create_request()
request_2 = self._create_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request_1)
storage.save_request(request_2)
requests = storage.load_requests()
self.assertEqual(2, len(requests))
self.assertEqual(request_1.id, requests[0].id)
self.assertEqual(request_2.id, requests[1].id)
self.assertIsNone(requests[0].response)
self.assertIsNone(requests[1].response)
def test_initialise_clears_old_folders(self):
test_dir = os.path.join(self.base_dir, '.seleniumwire', 'storage-test')
os.makedirs(test_dir)
two_days_ago = (datetime.now() - timedelta(days=2)).timestamp()
os.utime(test_dir, times=(two_days_ago, two_days_ago))
RequestStorage(base_dir=self.base_dir)
self.assertFalse(os.path.exists(test_dir))
def test_save_response_no_request(self):
request = self._create_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request)
response = self._create_response()
storage.clear_requests()
storage.save_response(request.id, response)
response_file_path = self._get_stored_path(request.id, 'response')
self.assertFalse(response_file_path)
def test_save_response(self):
mock_request = self._create_mock_request()
storage = RequestStorage(base_dir=self.base_dir)
request_id = storage.save_request(mock_request)
mock_response = self._create_mock_resonse()
storage.save_response(request_id, mock_response)
response_file_path = self._get_stored_path(request_id, 'response')
with open(response_file_path[0], 'rb') as loaded:
loaded_response = pickle.load(loaded)
self.assertEqual(loaded_response['status_code'], 200)
self.assertEqual(loaded_response['reason'], 'OK')
self.assertEqual(loaded_response['headers'], {
'Content-Type': 'application/json',
'Content-Length': '500'
})
def test_save_request(self):
mock_request = self._create_mock_request()
storage = RequestStorage(base_dir=self.base_dir)
request_id = storage.save_request(mock_request)
request_file_path = self._get_stored_path(request_id, 'request')
with open(request_file_path[0], 'rb') as loaded:
loaded_request = pickle.load(loaded)
self.assertEqual(loaded_request['id'], request_id)
self.assertEqual(loaded_request['path'], 'http://www.example.com/test/path/')
self.assertEqual(loaded_request['method'], 'GET')
self.assertEqual(loaded_request['headers'], {
'Host': 'www.example.com',
'Accept': '*/*'
})
self.assertIsNone(loaded_request['response'])
def test_save_request(self):
request = self._create_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request)
request_file_path = self._get_stored_path(request.id, 'request')
with open(request_file_path[0], 'rb') as loaded:
loaded_request = pickle.load(loaded)
self.assertEqual(request.id, loaded_request.id)
self.assertEqual('http://www.example.com/test/path/',
loaded_request.url)
self.assertEqual('GET', loaded_request.method)
self.assertEqual({
'Host': 'www.example.com',
'Accept': '*/*'
}, dict(loaded_request.headers))
self.assertIsNone(loaded_request.response)
def test_load_response(self):
request = self._create_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request)
response = self._create_response()
storage.save_response(request.id, response)
requests = storage.load_requests()
self.assertIsNotNone(requests[0].response)
def test_load_response_body(self):
body = b'test response body'
request = self._create_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request)
mock_response = self._create_response(body=body)
storage.save_response(request.id, mock_response)
response_body = storage.load_response_body(request.id)
self.assertEqual(body, response_body)
def test_clear_requests(self):
mock_request_1 = self._create_mock_request()
mock_request_2 = self._create_mock_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(mock_request_1)
storage.save_request(mock_request_2)
storage.clear_requests()
requests = storage.load_requests()
self.assertFalse(requests)
self.assertFalse(glob.glob(os.path.join(self.base_dir, '.seleniumwire', 'storage-*', '*')))
def test_load_response_encoded_body_error(self):
body = b'test response body'
request = self._create_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request)
response = self._create_response(body=body)
response.headers['Content-Encoding'] = 'gzip'
storage.save_response(request.id, response)
response_body = storage.load_requests()[0].response.body
self.assertEqual(body, response_body)
def test_load_last_request(self):
mock_request_1 = self._create_mock_request()
mock_request_2 = self._create_mock_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(mock_request_1)
request_id2 = storage.save_request(mock_request_2)
last_request = storage.load_last_request()
self.assertEqual(last_request['id'], request_id2)
def test_load_response(self):
mock_request = self._create_mock_request()
storage = RequestStorage(base_dir=self.base_dir)
request_id = storage.save_request(mock_request)
mock_response = self._create_mock_resonse()
storage.save_response(request_id, mock_response)
requests = storage.load_requests()
self.assertIsNotNone(requests[0]['response'])
def initialise(self, options):
"""Initialise this add-on with the selenium wire options.
This method must be called before the add-on starts handling requests.
Args:
options: The selenium wire options.
"""
self.options = options
self.storage = RequestStorage(
base_dir=options.get('request_storage_base_dir'))
# The logging in this add-on is not controlled by the logging
# in the selenium test because we're running in a subprocess.
# For the time being, configure basic logging using a config option.
logging.basicConfig(level=getattr(
logging, options.get('mitmproxy_log_level', 'ERROR')))
def test_load_response_encoded_body(self):
body = b'test response body'
io = BytesIO()
with gzip.GzipFile(fileobj=io, mode='wb') as f:
f.write(body)
request = self._create_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request)
response = self._create_response(body=io.getvalue())
response.headers['Content-Encoding'] = 'gzip'
storage.save_response(request.id, response)
response_body = storage.load_requests()[0].response.body
self.assertEqual(body, response_body)
def test_load_response_body_encoded(self):
io = BytesIO()
with gzip.GzipFile(fileobj=io, mode='wb') as f:
f.write(b'test response body')
mock_request = self._create_mock_request()
storage = RequestStorage(base_dir=self.base_dir)
request_id = storage.save_request(mock_request, request_body=b'test request body')
mock_response = self._create_mock_resonse()
mock_response.headers['Content-Encoding'] = 'gzip'
storage.save_response(request_id, mock_response, response_body=io.getvalue())
response_body = storage.load_response_body(request_id)
self.assertEqual(response_body, b'test response body')
def test_save_response_with_body(self):
body = b'some response body'
request = self._create_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request)
response = self._create_response(body=body)
storage.save_response(request.id, response)
response_file_path = self._get_stored_path(request.id, 'response')
with open(response_file_path[0], 'rb') as loaded:
loaded_response = pickle.load(loaded)
self.assertEqual(b'some response body', loaded_response.body)
def test_save_response(self):
request = self._create_request()
storage = RequestStorage(base_dir=self.base_dir)
storage.save_request(request)
response = self._create_response()
storage.save_response(request.id, response)
response_file_path = self._get_stored_path(request.id, 'response')
with open(response_file_path[0], 'rb') as loaded:
loaded_response = pickle.load(loaded)
self.assertEqual(200, loaded_response.status_code)
self.assertEqual('OK', loaded_response.reason)
self.assertEqual(
{
'Content-Type': 'application/json',
'Content-Length': '500'
}, dict(loaded_response.headers))
def test_find(self):
mock_request_1 = self._create_mock_request(
'http://www.example.com/test/path/?foo=bar')
mock_request_2 = self._create_mock_request(
'http://www.stackoverflow.com/other/path/?x=y')
mock_response = self._create_mock_resonse()
storage = RequestStorage(base_dir=self.base_dir)
request_id = storage.save_request(mock_request_1)
storage.save_response(request_id, mock_response)
storage.save_request(mock_request_2)
self.assertEqual(storage.find('/test/path/')['id'], request_id)
self.assertEqual(storage.find('/test/path/?foo=bar')['id'], request_id)
self.assertEqual(
storage.find('http://www.example.com/test/path/?foo=bar')['id'],
request_id)
self.assertEqual(
storage.find('http://www.example.com/test/path/')['id'],
request_id)
self.assertIsNone(storage.find('/different/path'))
self.assertIsNone(storage.find('/test/path/?x=y'))
self.assertIsNone(
storage.find('http://www.example.com/different/path/?foo=bar'))
self.assertIsNone(
storage.find('http://www.different.com/test/path/?foo=bar'))
self.assertIsNone(
storage.find('http://www.example.com/test/path/?x=y'))
def test_load_last_request_none(self):
storage = RequestStorage(base_dir=self.base_dir)
last_request = storage.load_last_request()
self.assertIsNone(last_request)
def test_initialise(self):
RequestStorage(base_dir=self.base_dir)
storage_dir = glob.glob(
os.path.join(self.base_dir, '.seleniumwire', 'storage-*'))
self.assertEqual(len(storage_dir), 1)
