def setUp(self):
self.archive = httparchive.HttpArchive()
self.archive[self.REQUEST] = self.RESPONSE
# Also add an identical POST request for testing
request = httparchive.ArchivedHttpRequest('POST', 'www.test.com', '/',
None, self.REQUEST_HEADERS)
self.archive[request] = self.RESPONSE
def testFetch(self):
request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/',
None, {})
response = self.createTestResponse()
archive = httparchive.HttpArchive()
archive[request] = response
fetch = httpclient.ReplayHttpArchiveFetch(archive, None,
self.dummy_injector)
self.checkTestResponse(fetch(request), archive, request)
def testFetch(self, real_http_fetch):
http_fetch_instance = real_http_fetch.return_value
response = self.createTestResponse()
http_fetch_instance.return_value = response
archive = httparchive.HttpArchive()
fetch = httpclient.RecordHttpArchiveFetch(archive, self.dummy_injector)
request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/',
None, {})
self.checkTestResponse(fetch(request), archive, request)
def setup_find_closest_request(self):
headers = {}
request1 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/a?hello=world', None, headers)
request2 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/a?foo=bar', None, headers)
request3 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/b?hello=world', None, headers)
archive = httparchive.HttpArchive()
# Add requests 2 and 3 and find closest match with request1
archive[request2] = self.RESPONSE
archive[request3] = self.RESPONSE
return archive, request1, request2, request3
def testInjectedDate(self, os_path, util_exists, util_resource_string):
os_path.return_value = False
util_exists.return_value = True
util_resource_string.return_value = \
["""var time_seed={}""".format(script_injector.TIME_SEED_MARKER)]
request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/',
None, {})
response = self.createTestResponse()
archive = httparchive.HttpArchive()
archive[request] = response
fetch = httpclient.ReplayHttpArchiveFetch(
archive, None, script_injector.GetScriptInjector("time_script.js"))
self.assertEqual(
['<script>var time_seed=1479344523000</script><body>test</body>'],
fetch(request).response_data)
def convert_to_wpr(har):
# Assumes one page load per har file.
try:
archive = httparchive.HttpArchive()
for entry in har["log"]["entries"]:
timings = entry["timings"]
if is_favicon(entry):
# Favicons don't affect page load time.
continue
maybe_fill_in_image_data(entry)
# TODO(cs): find a better way to infer ssl
is_ssl = "ssl" in timings and timings["ssl"] != -1
request = convert_request(entry["request"], is_ssl)
response = convert_response(entry["response"], timings)
archive[request] = response
return archive
except KeyError as e:
traceback.print_exc(e)
raise ValueError("Malformed HAR. " + str(e))
def test_find_closest_request_timestamp(self):
headers = {}
request1 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/index.html?time=100000000&important=true',
None, headers)
request2 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/index.html?time=99999999&important=true',
None, headers)
request3 = httparchive.ArchivedHttpRequest(
'GET', 'www.test.com', '/index.html?time=10000000&important=false',
None, headers)
archive = httparchive.HttpArchive()
# Add requests 2 and 3 and find closest match with request1
archive[request2] = self.RESPONSE
archive[request3] = self.RESPONSE
# Although request3 is lexicographically closer, request2 is semantically
# more similar.
self.assertEqual(request2,
archive.find_closest_request(request1, use_path=True))
def replay(options, replay_filename):
if options.admin_check and options.IsRootRequired():
platformsettings.rerun_as_administrator()
configure_logging(options.log_level, options.log_file)
server_manager = servermanager.ServerManager(options.record)
cache_misses = None
if options.cache_miss_file:
if os.path.exists(options.cache_miss_file):
logging.warning(
'Cache Miss Archive file %s already exists; '
'replay will load and append entries to archive file',
options.cache_miss_file)
cache_misses = cachemissarchive.CacheMissArchive.Load(
options.cache_miss_file)
else:
cache_misses = cachemissarchive.CacheMissArchive(
options.cache_miss_file)
if options.server:
AddDnsForward(server_manager, options.server)
else:
real_dns_lookup = dnsproxy.RealDnsLookup(
name_servers=[platformsettings.get_original_primary_nameserver()])
if options.record:
httparchive.HttpArchive.AssertWritable(replay_filename)
if options.append and os.path.exists(replay_filename):
http_archive = httparchive.HttpArchive.Load(replay_filename)
logging.info('Appending to %s (loaded %d existing responses)',
replay_filename, len(http_archive))
else:
http_archive = httparchive.HttpArchive()
else:
http_archive = httparchive.HttpArchive.Load(replay_filename)
logging.info('Loaded %d responses from %s', len(http_archive),
replay_filename)
server_manager.AppendRecordCallback(real_dns_lookup.ClearCache)
server_manager.AppendRecordCallback(http_archive.clear)
ipfw_dns_host = None
if options.dns_forwarding or options.shaping_dummynet:
# compute the ip/host used for the DNS server and traffic shaping
ipfw_dns_host = options.host
if not ipfw_dns_host:
ipfw_dns_host = platformsettings.get_server_ip_address(
options.server_mode)
if options.dns_forwarding:
if not options.server_mode and ipfw_dns_host == '127.0.0.1':
AddDnsForward(server_manager, ipfw_dns_host)
AddDnsProxy(server_manager, options, ipfw_dns_host,
options.dns_port, real_dns_lookup, http_archive)
if options.ssl and options.https_root_ca_cert_path is None:
options.https_root_ca_cert_path = os.path.join(
os.path.dirname(__file__), 'wpr_cert.pem')
http_proxy_address = options.host
if not http_proxy_address:
http_proxy_address = platformsettings.get_httpproxy_ip_address(
options.server_mode)
AddWebProxy(server_manager, options, http_proxy_address,
real_dns_lookup, http_archive, cache_misses)
AddTrafficShaper(server_manager, options, ipfw_dns_host)
exit_status = 0
try:
server_manager.Run()
except KeyboardInterrupt:
logging.info('Shutting down.')
except (dnsproxy.DnsProxyException, trafficshaper.TrafficShaperException,
platformsettings.NotAdministratorError,
platformsettings.DnsUpdateError) as e:
logging.critical('%s: %s', e.__class__.__name__, e)
exit_status = 1
except:
logging.critical(traceback.format_exc())
exit_status = 2
if options.record:
http_archive.Persist(replay_filename)
logging.info('Saved %d responses to %s', len(http_archive),
replay_filename)
if cache_misses:
cache_misses.Persist()
logging.info('Saved %d cache misses and %d requests to %s',
cache_misses.get_total_cache_misses(),
len(cache_misses.request_counts.keys()),
options.cache_miss_file)
return exit_status
def test_init(self):
archive = httparchive.HttpArchive()
self.assertEqual(len(archive), 0)
def replay(options, replay_filename):
if options.record and sys.version_info < (2, 7, 9):
print ('Need Python 2.7.9 or greater for recording mode.\n'
'For instructions on how to upgrade Python on Ubuntu 14.04, see:\n'
'http://mbless.de/blog/2016/01/09/upgrade-to-python-2711-on-ubuntu-1404-lts.html\n')
if options.admin_check and options.IsRootRequired():
platformsettings.rerun_as_administrator()
configure_logging(options.log_level, options.log_file)
server_manager = servermanager.ServerManager(options.record)
if options.server:
AddDnsForward(server_manager, options.server)
else:
if options.record:
httparchive.HttpArchive.AssertWritable(replay_filename)
if options.append and os.path.exists(replay_filename):
http_archive = httparchive.HttpArchive.Load(replay_filename)
logging.info('Appending to %s (loaded %d existing responses)',
replay_filename, len(http_archive))
else:
http_archive = httparchive.HttpArchive()
else:
http_archive = httparchive.HttpArchive.Load(replay_filename)
logging.info('Loaded %d responses from %s',
len(http_archive), replay_filename)
server_manager.AppendRecordCallback(http_archive.clear)
ipfw_dns_host = None
if options.dns_forwarding or options.shaping_dummynet:
# compute the ip/host used for the DNS server and traffic shaping
ipfw_dns_host = options.host
if not ipfw_dns_host:
ipfw_dns_host = platformsettings.get_server_ip_address(
options.server_mode)
real_dns_lookup = dnsproxy.RealDnsLookup(
name_servers=[platformsettings.get_original_primary_nameserver()],
dns_forwarding=options.dns_forwarding,
proxy_host=ipfw_dns_host,
proxy_port=options.dns_port)
server_manager.AppendRecordCallback(real_dns_lookup.ClearCache)
if options.dns_forwarding:
if not options.server_mode and ipfw_dns_host == '127.0.0.1':
AddDnsForward(server_manager, ipfw_dns_host)
AddDnsProxy(server_manager, options, ipfw_dns_host, options.dns_port,
real_dns_lookup, http_archive)
if options.ssl and options.https_root_ca_cert_path is None:
options.https_root_ca_cert_path = os.path.join(os.path.dirname(__file__),
'wpr_cert.pem')
http_proxy_address = options.host
if not http_proxy_address:
http_proxy_address = platformsettings.get_httpproxy_ip_address(
options.server_mode)
AddWebProxy(server_manager, options, http_proxy_address, real_dns_lookup,
http_archive)
AddTrafficShaper(server_manager, options, ipfw_dns_host)
exit_status = 0
try:
server_manager.Run()
except KeyboardInterrupt:
logging.info('Shutting down.')
except (dnsproxy.DnsProxyException,
trafficshaper.TrafficShaperException,
platformsettings.NotAdministratorError,
platformsettings.DnsUpdateError) as e:
logging.critical('%s: %s', e.__class__.__name__, e)
exit_status = 1
except Exception:
logging.critical(traceback.format_exc())
exit_status = 2
if options.record:
http_archive.Persist(replay_filename)
logging.info('Saved %d responses to %s', len(http_archive), replay_filename)
return exit_status
def replay(options, replay_filename):
platform_settings = platformsettings.get_platform_settings()
if options.IsRootRequired():
platform_settings.rerun_as_administrator()
configure_logging(platform_settings, options.log_level, options.log_file)
server_manager = servermanager.ServerManager(options.record)
cache_misses = None
if options.cache_miss_file:
if os.path.exists(options.cache_miss_file):
logging.warning('Cache Miss Archive file %s already exists; '
'replay will load and append entries to archive file',
options.cache_miss_file)
cache_misses = cachemissarchive.CacheMissArchive.Load(
options.cache_miss_file)
else:
cache_misses = cachemissarchive.CacheMissArchive(
options.cache_miss_file)
if options.server:
AddDnsForward(server_manager, platform_settings, options.server)
else:
host = platform_settings.get_server_ip_address(options.server_mode)
real_dns_lookup = dnsproxy.RealDnsLookup(
name_servers=[platform_settings.get_original_primary_dns()])
if options.record:
http_archive = httparchive.HttpArchive()
http_archive.AssertWritable(replay_filename)
else:
http_archive = httparchive.HttpArchive.Load(replay_filename)
logging.info('Loaded %d responses from %s',
len(http_archive), replay_filename)
server_manager.AppendRecordCallback(real_dns_lookup.ClearCache)
server_manager.AppendRecordCallback(http_archive.clear)
if options.dns_forwarding:
if not options.server_mode:
AddDnsForward(server_manager, platform_settings, host)
AddDnsProxy(server_manager, options, host, real_dns_lookup, http_archive)
if options.ssl and options.certfile is None:
options.certfile = platform_settings.get_certfile_name()
server_manager.AppendStartStopFunctions(
[platform_settings.create_certfile, options.certfile],
[os.unlink, options.certfile])
AddWebProxy(server_manager, options, host, real_dns_lookup,
http_archive, cache_misses)
AddTrafficShaper(server_manager, options, host)
exit_status = 0
try:
server_manager.Run()
except KeyboardInterrupt:
logging.info('Shutting down.')
except (dnsproxy.DnsProxyException,
trafficshaper.TrafficShaperException,
platformsettings.NotAdministratorError,
platformsettings.DnsUpdateError) as e:
logging.critical('%s: %s', e.__class__.__name__, e)
exit_status = 1
except:
logging.critical(traceback.format_exc())
exit_status = 2
if options.record:
http_archive.Persist(replay_filename)
logging.info('Saved %d responses to %s', len(http_archive), replay_filename)
if cache_misses:
cache_misses.Persist()
logging.info('Saved %d cache misses and %d requests to %s',
cache_misses.get_total_cache_misses(),
len(cache_misses.request_counts.keys()),
options.cache_miss_file)
return exit_status
