def test_record_request(self):
request = self.REQUEST
cache_archive = cachemissarchive.CacheMissArchive('empty-archive')
self.assertEqual(len(cache_archive.request_counts), 0)
cache_archive.record_request(request,
is_record_mode=True,
is_cache_miss=False)
self.assertEqual(len(cache_archive.request_counts), 1)
self.assertEqual(cache_archive.request_counts[request], (1, 0))
cache_archive.record_request(request,
is_record_mode=False,
is_cache_miss=False)
self.assertEqual(len(cache_archive.request_counts), 1)
self.assertEqual(cache_archive.request_counts[request], (1, 1))
def load_mock_archive(self):
self.cache_archive = cachemissarchive.CacheMissArchive('mock-archive')
self.num_requests = 0
urls_list = [
'http://www.zappos.com/',
'http://www.msn.com/',
'http://www.amazon.com/',
'http://www.google.com/',
]
self.cache_archive.set_urls_list(urls_list)
for line in get_mock_requests():
# Each line contains: (command, host, path, request_body, headers)
# Delimited by '%'
args = line.split('%')
headers = ast.literal_eval(args[4].strip('\n '))
request = ArchivedHttpRequest(args[0], args[1], args[2], args[3],
headers)
self.cache_archive.record_request(request,
is_record_mode=False,
is_cache_miss=True)
self.num_requests += 1
def replay(options, replay_filename):
if options.admin_check and options.IsRootRequired():
platformsettings.rerun_as_administrator()
configure_logging(options.log_level, options.log_file)
server_manager = servermanager.ServerManager(options.record)
cache_misses = None
if options.cache_miss_file:
if os.path.exists(options.cache_miss_file):
logging.warning(
'Cache Miss Archive file %s already exists; '
'replay will load and append entries to archive file',
options.cache_miss_file)
cache_misses = cachemissarchive.CacheMissArchive.Load(
options.cache_miss_file)
else:
cache_misses = cachemissarchive.CacheMissArchive(
options.cache_miss_file)
if options.server:
AddDnsForward(server_manager, options.server)
else:
real_dns_lookup = dnsproxy.RealDnsLookup(
name_servers=[platformsettings.get_original_primary_nameserver()])
if options.record:
httparchive.HttpArchive.AssertWritable(replay_filename)
if options.append and os.path.exists(replay_filename):
http_archive = httparchive.HttpArchive.Load(replay_filename)
logging.info('Appending to %s (loaded %d existing responses)',
replay_filename, len(http_archive))
else:
http_archive = httparchive.HttpArchive()
else:
http_archive = httparchive.HttpArchive.Load(replay_filename)
logging.info('Loaded %d responses from %s', len(http_archive),
replay_filename)
server_manager.AppendRecordCallback(real_dns_lookup.ClearCache)
server_manager.AppendRecordCallback(http_archive.clear)
ipfw_dns_host = None
if options.dns_forwarding or options.shaping_dummynet:
# compute the ip/host used for the DNS server and traffic shaping
ipfw_dns_host = options.host
if not ipfw_dns_host:
ipfw_dns_host = platformsettings.get_server_ip_address(
options.server_mode)
if options.dns_forwarding:
if not options.server_mode and ipfw_dns_host == '127.0.0.1':
AddDnsForward(server_manager, ipfw_dns_host)
AddDnsProxy(server_manager, options, ipfw_dns_host,
options.dns_port, real_dns_lookup, http_archive)
if options.ssl and options.https_root_ca_cert_path is None:
options.https_root_ca_cert_path = os.path.join(
os.path.dirname(__file__), 'wpr_cert.pem')
http_proxy_address = options.host
if not http_proxy_address:
http_proxy_address = platformsettings.get_httpproxy_ip_address(
options.server_mode)
AddWebProxy(server_manager, options, http_proxy_address,
real_dns_lookup, http_archive, cache_misses)
AddTrafficShaper(server_manager, options, ipfw_dns_host)
exit_status = 0
try:
server_manager.Run()
except KeyboardInterrupt:
logging.info('Shutting down.')
except (dnsproxy.DnsProxyException, trafficshaper.TrafficShaperException,
platformsettings.NotAdministratorError,
platformsettings.DnsUpdateError) as e:
logging.critical('%s: %s', e.__class__.__name__, e)
exit_status = 1
except:
logging.critical(traceback.format_exc())
exit_status = 2
if options.record:
http_archive.Persist(replay_filename)
logging.info('Saved %d responses to %s', len(http_archive),
replay_filename)
if cache_misses:
cache_misses.Persist()
logging.info('Saved %d cache misses and %d requests to %s',
cache_misses.get_total_cache_misses(),
len(cache_misses.request_counts.keys()),
options.cache_miss_file)
return exit_status
def replay(options, replay_filename):
platform_settings = platformsettings.get_platform_settings()
if options.IsRootRequired():
platform_settings.rerun_as_administrator()
configure_logging(platform_settings, options.log_level, options.log_file)
server_manager = servermanager.ServerManager(options.record)
cache_misses = None
if options.cache_miss_file:
if os.path.exists(options.cache_miss_file):
logging.warning('Cache Miss Archive file %s already exists; '
'replay will load and append entries to archive file',
options.cache_miss_file)
cache_misses = cachemissarchive.CacheMissArchive.Load(
options.cache_miss_file)
else:
cache_misses = cachemissarchive.CacheMissArchive(
options.cache_miss_file)
if options.server:
AddDnsForward(server_manager, platform_settings, options.server)
else:
host = platform_settings.get_server_ip_address(options.server_mode)
real_dns_lookup = dnsproxy.RealDnsLookup(
name_servers=[platform_settings.get_original_primary_dns()])
if options.record:
http_archive = httparchive.HttpArchive()
http_archive.AssertWritable(replay_filename)
else:
http_archive = httparchive.HttpArchive.Load(replay_filename)
logging.info('Loaded %d responses from %s',
len(http_archive), replay_filename)
server_manager.AppendRecordCallback(real_dns_lookup.ClearCache)
server_manager.AppendRecordCallback(http_archive.clear)
if options.dns_forwarding:
if not options.server_mode:
AddDnsForward(server_manager, platform_settings, host)
AddDnsProxy(server_manager, options, host, real_dns_lookup, http_archive)
if options.ssl and options.certfile is None:
options.certfile = platform_settings.get_certfile_name()
server_manager.AppendStartStopFunctions(
[platform_settings.create_certfile, options.certfile],
[os.unlink, options.certfile])
AddWebProxy(server_manager, options, host, real_dns_lookup,
http_archive, cache_misses)
AddTrafficShaper(server_manager, options, host)
exit_status = 0
try:
server_manager.Run()
except KeyboardInterrupt:
logging.info('Shutting down.')
except (dnsproxy.DnsProxyException,
trafficshaper.TrafficShaperException,
platformsettings.NotAdministratorError,
platformsettings.DnsUpdateError) as e:
logging.critical('%s: %s', e.__class__.__name__, e)
exit_status = 1
except:
logging.critical(traceback.format_exc())
exit_status = 2
if options.record:
http_archive.Persist(replay_filename)
logging.info('Saved %d responses to %s', len(http_archive), replay_filename)
if cache_misses:
cache_misses.Persist()
logging.info('Saved %d cache misses and %d requests to %s',
cache_misses.get_total_cache_misses(),
len(cache_misses.request_counts.keys()),
options.cache_miss_file)
return exit_status
def test_get_total_referers_small(self):
cache_archive = cachemissarchive.CacheMissArchive('empty-archive')
self.assertEqual(cache_archive.get_total_referers(), 0)
referer = 'mock_referer'
cache_archive.record_cache_miss(self.REQUEST, page_url=referer)
self.assertEqual(cache_archive.get_total_referers(), 1)
def test_record_cache_miss(self):
cache_archive = cachemissarchive.CacheMissArchive('empty-archive')
referer = 'mock_referer'
cache_archive.record_cache_miss(self.REQUEST, page_url=referer)
self.assert_(cache_archive.archive[referer])
def test_init(self):
empty_archive = cachemissarchive.CacheMissArchive('empty-archive')
self.assert_(not empty_archive.archive)
