def test_matches(self):
headers = {}
request1 = httparchive.ArchivedHttpRequest('GET', 'www.test.com',
'/index.html?hello=world',
None, headers)
request2 = httparchive.ArchivedHttpRequest('GET', 'www.test.com',
'/index.html?foo=bar', None,
headers)
self.assert_(not request1.matches(
request2.command, request2.host, request2.path, use_query=True))
self.assert_(
request1.matches(request2.command,
request2.host,
request2.path,
use_query=False))
self.assert_(
request1.matches(request2.command,
request2.host,
None,
use_query=True))
self.assert_(
request1.matches(request2.command,
None,
request2.path,
use_query=False))
empty_request = httparchive.ArchivedHttpRequest(
None, None, None, None, headers)
self.assert_(not empty_request.matches(
request2.command, request2.host, None, use_query=True))
self.assert_(not empty_request.matches(
request2.command, None, request2.path, use_query=False))
def setup_find_closest_request(self):
headers = {}
request1 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/a?hello=world', None, headers)
request2 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/a?foo=bar', None, headers)
request3 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/b?hello=world', None, headers)
archive = httparchive.HttpArchive()
# Add requests 2 and 3 and find closest match with request1
archive[request2] = self.RESPONSE
archive[request3] = self.RESPONSE
return archive, request1, request2, request3
def test_get_unmodified_headers(self):
request = self.REQUEST
response = self.RESPONSE
archive = self.archive
not_modified_response = httparchive.create_response(304)
# Succeed check
request_headers = {'if-unmodified-since': self.DATE_PAST}
request = create_request(request_headers)
self.assertEqual(archive.get(request), not_modified_response)
# Fail check
request_headers = {'if-unmodified-since': self.DATE_FUTURE}
request = create_request(request_headers)
self.assertEqual(archive.get(request), response)
# Succeed check
request_headers = {'if-unmodified-since': self.DATE_PRESENT}
request = create_request(request_headers)
self.assertEqual(archive.get(request), not_modified_response)
# Fail check
request_headers = {'if-unmodified-since': self.DATE_INVALID}
request = create_request(request_headers)
self.assertEqual(archive.get(request), response)
# Fail check since the request is not a GET or HEAD request (as per RFC)
request_headers = {'if-modified-since': self.DATE_PAST}
request = httparchive.ArchivedHttpRequest('POST', 'www.test.com', '/',
None, request_headers)
self.assertEqual(archive.get(request), response)
def setUp(self):
self.archive = httparchive.HttpArchive()
self.archive[self.REQUEST] = self.RESPONSE
# Also add an identical POST request for testing
request = httparchive.ArchivedHttpRequest('POST', 'www.test.com', '/',
None, self.REQUEST_HEADERS)
self.archive[request] = self.RESPONSE
def test_get_cmp_seq(self):
# The order of key-value pairs in query and header respectively should not
# matter.
headers = {'k2': 'v2', 'k1': 'v1'}
request = httparchive.ArchivedHttpRequest('GET', 'www.test.com',
'/a?c=d&a=b;e=f', None,
headers)
self.assertEqual([('a', 'b'), ('c', 'd'), ('e', 'f'), ('k1', 'v1'),
('k2', 'v2')], request._GetCmpSeq('c=d&a=b;e=f'))
def testFetch(self):
request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/',
None, {})
response = self.createTestResponse()
archive = httparchive.HttpArchive()
archive[request] = response
fetch = httpclient.ReplayHttpArchiveFetch(archive, None,
self.dummy_injector)
self.checkTestResponse(fetch(request), archive, request)
def testFetch(self, real_http_fetch):
http_fetch_instance = real_http_fetch.return_value
response = self.createTestResponse()
http_fetch_instance.return_value = response
archive = httparchive.HttpArchive()
fetch = httpclient.RecordHttpArchiveFetch(archive, self.dummy_injector)
request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/',
None, {})
self.checkTestResponse(fetch(request), archive, request)
def testFetchSSLRequest(self):
real_dns_lookup = dnsproxy.RealDnsLookup(
name_servers=[platformsettings.get_original_primary_nameserver()])
fetch = httpclient.RealHttpFetch(real_dns_lookup)
request = httparchive.ArchivedHttpRequest(
command='GET', host='google.com', full_path='/search?q=dogs',
request_body=None, headers={}, is_ssl=True)
response = fetch(request)
self.assertIsNotNone(response)
def test_find_closest_request_timestamp(self):
headers = {}
request1 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/index.html?time=100000000&important=true',
None, headers)
request2 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/index.html?time=99999999&important=true',
None, headers)
request3 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/index.html?time=10000000&important=false',
None, headers)
archive = httparchive.HttpArchive()
# Add requests 2 and 3 and find closest match with request1
archive[request2] = self.RESPONSE
archive[request3] = self.RESPONSE
# Although request3 is lexicographically closer, request2 is semantically
# more similar.
self.assertEqual(request2,
archive.find_closest_request(request1, use_path=True))
def request_handler(self, method, uri, hdrs, res_start, req_pause):
"""
Based on method, host and uri to fetch the matching response and reply
to browser using spdy.
"""
dummy = http_common.dummy
def simple_responder(code, content):
res_hdrs = [('content-type', 'text/html'), ('version', 'HTTP/1.1')]
res_body, res_done = res_start(str(code), content, res_hdrs, dummy)
res_body(None)
res_done(None)
host = ''
for name, value in hdrs:
if name.lower() == 'host':
host = value
self.log.debug("request: %s, uri: %s, method: %s", host, uri, method)
if method == 'GET':
request = httparchive.ArchivedHttpRequest(method, host, uri, None,
dict(hdrs))
response_code = self.custom_handlers.handle(request)
if response_code:
simple_responder(response_code, "Handled by custom handlers")
return dummy, dummy
response = self.http_archive_fetch(request)
if response:
res_hdrs = [('version', 'HTTP/1.1')]
for name, value in response.headers:
name_lower = name.lower()
if name_lower == CONTENT_LENGTH:
res_hdrs.append((name, str(value)))
elif name_lower in (STATUS, VERSION):
pass
else:
res_hdrs.append((name_lower, value))
res_body, res_done = res_start(str(response.status),
response.reason, res_hdrs,
dummy)
body = ''
for item in response.response_data:
res_body(item)
res_done(None)
else:
self.log.error("404 returned: %s %s", method, uri)
simple_responder(404, "file not found")
else:
# TODO(lzheng): Add support for other methods.
self.log.error("method: %s is not supported: %s", method, uri)
simple_responder(500, "Not supported")
return dummy, dummy
def get_archived_http_request(self):
host = self.headers.get('host')
if host is None:
logging.error('Request without host header')
self.send_error(500)
return None
parsed = urlparse.urlparse(self.path)
params = ';%s' % parsed.params if parsed.params else ''
query = '?%s' % parsed.query if parsed.query else ''
fragment = '#%s' % parsed.fragment if parsed.fragment else ''
full_path = '%s%s%s%s' % (parsed.path, params, query, fragment)
repr_path = full_path
# remove all designated groups from the matched URL.
# e.g if the pattern is "(.*\.)?foo.com/bar.*(qux=1&).*"
# then "abc.foo.com/bart?qux=1&z" --> "foo.com/bart?z"
for path in self.server.paths_to_edit:
groups = path.match('%s%s' % (host, repr_path))
if groups:
start_include = 0
new_path = ''
for i in xrange(groups.lastindex):
start_group, end_group = groups.span(i + 1)
new_path += repr_path[start_include:start_group]
start_include = end_group
repr_path = new_path + repr_path[start_include::]
logging.info(
'While %sing replaced %s with %s',
('record' if self.server.http_archive_fetch.is_record_mode
else 'replay'), full_path, repr_path)
exclude_headers = []
for path, undesirable_key in self.server.undesirable_headers.items():
if re.match(r'%s' % path, parsed.path):
exclude_headers.append(undesirable_key)
for path, status in self.server.error_paths:
if path.match('%s%s' % (host, full_path)):
logging.debug('Send %d for %s%s', status, host, full_path)
logging.debug(path.pattern)
self.send_error(status)
return None
return httparchive.ArchivedHttpRequest(self.command, host, full_path,
self.read_request_body(),
self.get_header_dict(),
self.server.is_ssl, repr_path,
exclude_headers)
def convert_request(request, is_ssl):
command = convert_unicode(request["method"])
url = convert_unicode(request["url"])
url_object = urlparse.urlsplit(url)
host = url_object.netloc
# TODO(cs): must be a more robust way to get the full path. urlparse
# doesn't seem to have it though.
full_path = url[url.find(host) + len(host):]
request_body = None
if "postData" in request:
request_body = convert_unicode(request["postData"])
headers = convert_headers_to_dict(request["headers"])
return httparchive.ArchivedHttpRequest(command, host, full_path,
request_body, headers,
is_ssl=is_ssl)
def get_archived_http_request(self):
host = self.headers.get('host')
if host is None:
logging.error('Request without host header')
return None
parsed = urlparse.urlparse(self.path)
query = '?%s' % parsed.query if parsed.query else ''
fragment = '#%s' % parsed.fragment if parsed.fragment else ''
full_path = '%s%s%s' % (parsed.path, query, fragment)
return httparchive.ArchivedHttpRequest(self.command, host, full_path,
self.read_request_body(),
self.get_header_dict(),
self.server.is_ssl)
def testInjectedDate(self, os_path, util_exists, util_resource_string):
os_path.return_value = False
util_exists.return_value = True
util_resource_string.return_value = \
["""var time_seed={}""".format(script_injector.TIME_SEED_MARKER)]
request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/',
None, {})
response = self.createTestResponse()
archive = httparchive.HttpArchive()
archive[request] = response
fetch = httpclient.ReplayHttpArchiveFetch(
archive, None, script_injector.GetScriptInjector("time_script.js"))
self.assertEqual(
['<script>var time_seed=1479344523000</script><body>test</body>'],
fetch(request).response_data)
def get_archived_http_request(self):
host = self.headers.get('host')
if host is None:
logging.error('Request without host header')
return None
parsed = urlparse.urlparse(self.path)
params = ';%s' % parsed.params if parsed.params else ''
query = '?%s' % parsed.query if parsed.query else ''
fragment = '#%s' % parsed.fragment if parsed.fragment else ''
full_path = '%s%s%s%s' % (parsed.path, params, query, fragment)
StubRequest = collections.namedtuple('StubRequest',
('host', 'full_path'))
request, response = StubRequest(host, full_path), None
self.server.log_url(request, response)
return httparchive.ArchivedHttpRequest(self.command, host, full_path,
self.read_request_body(),
self.get_header_dict(),
self.server.is_ssl)
def create_request(headers):
return httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/', None,
headers)
