def test_record_request(self): request = self.REQUEST cache_archive = cachemissarchive.CacheMissArchive('empty-archive') self.assertEqual(len(cache_archive.request_counts), 0) cache_archive.record_request(request, is_record_mode=True, is_cache_miss=False) self.assertEqual(len(cache_archive.request_counts), 1) self.assertEqual(cache_archive.request_counts[request], (1, 0)) cache_archive.record_request(request, is_record_mode=False, is_cache_miss=False) self.assertEqual(len(cache_archive.request_counts), 1) self.assertEqual(cache_archive.request_counts[request], (1, 1))
def load_mock_archive(self): self.cache_archive = cachemissarchive.CacheMissArchive('mock-archive') self.num_requests = 0 urls_list = [ 'http://www.zappos.com/', 'http://www.msn.com/', 'http://www.amazon.com/', 'http://www.google.com/', ] self.cache_archive.set_urls_list(urls_list) for line in get_mock_requests(): # Each line contains: (command, host, path, request_body, headers) # Delimited by '%' args = line.split('%') headers = ast.literal_eval(args[4].strip('\n ')) request = ArchivedHttpRequest(args[0], args[1], args[2], args[3], headers) self.cache_archive.record_request(request, is_record_mode=False, is_cache_miss=True) self.num_requests += 1
def replay(options, replay_filename): if options.admin_check and options.IsRootRequired(): platformsettings.rerun_as_administrator() configure_logging(options.log_level, options.log_file) server_manager = servermanager.ServerManager(options.record) cache_misses = None if options.cache_miss_file: if os.path.exists(options.cache_miss_file): logging.warning( 'Cache Miss Archive file %s already exists; ' 'replay will load and append entries to archive file', options.cache_miss_file) cache_misses = cachemissarchive.CacheMissArchive.Load( options.cache_miss_file) else: cache_misses = cachemissarchive.CacheMissArchive( options.cache_miss_file) if options.server: AddDnsForward(server_manager, options.server) else: real_dns_lookup = dnsproxy.RealDnsLookup( name_servers=[platformsettings.get_original_primary_nameserver()]) if options.record: httparchive.HttpArchive.AssertWritable(replay_filename) if options.append and os.path.exists(replay_filename): http_archive = httparchive.HttpArchive.Load(replay_filename) logging.info('Appending to %s (loaded %d existing responses)', replay_filename, len(http_archive)) else: http_archive = httparchive.HttpArchive() else: http_archive = httparchive.HttpArchive.Load(replay_filename) logging.info('Loaded %d responses from %s', len(http_archive), replay_filename) server_manager.AppendRecordCallback(real_dns_lookup.ClearCache) server_manager.AppendRecordCallback(http_archive.clear) ipfw_dns_host = None if options.dns_forwarding or options.shaping_dummynet: # compute the ip/host used for the DNS server and traffic shaping ipfw_dns_host = options.host if not ipfw_dns_host: ipfw_dns_host = platformsettings.get_server_ip_address( options.server_mode) if options.dns_forwarding: if not options.server_mode and ipfw_dns_host == '127.0.0.1': AddDnsForward(server_manager, ipfw_dns_host) AddDnsProxy(server_manager, options, ipfw_dns_host, options.dns_port, real_dns_lookup, http_archive) if options.ssl and options.https_root_ca_cert_path is None: options.https_root_ca_cert_path = os.path.join( os.path.dirname(__file__), 'wpr_cert.pem') http_proxy_address = options.host if not http_proxy_address: http_proxy_address = platformsettings.get_httpproxy_ip_address( options.server_mode) AddWebProxy(server_manager, options, http_proxy_address, real_dns_lookup, http_archive, cache_misses) AddTrafficShaper(server_manager, options, ipfw_dns_host) exit_status = 0 try: server_manager.Run() except KeyboardInterrupt: logging.info('Shutting down.') except (dnsproxy.DnsProxyException, trafficshaper.TrafficShaperException, platformsettings.NotAdministratorError, platformsettings.DnsUpdateError) as e: logging.critical('%s: %s', e.__class__.__name__, e) exit_status = 1 except: logging.critical(traceback.format_exc()) exit_status = 2 if options.record: http_archive.Persist(replay_filename) logging.info('Saved %d responses to %s', len(http_archive), replay_filename) if cache_misses: cache_misses.Persist() logging.info('Saved %d cache misses and %d requests to %s', cache_misses.get_total_cache_misses(), len(cache_misses.request_counts.keys()), options.cache_miss_file) return exit_status
def replay(options, replay_filename): platform_settings = platformsettings.get_platform_settings() if options.IsRootRequired(): platform_settings.rerun_as_administrator() configure_logging(platform_settings, options.log_level, options.log_file) server_manager = servermanager.ServerManager(options.record) cache_misses = None if options.cache_miss_file: if os.path.exists(options.cache_miss_file): logging.warning('Cache Miss Archive file %s already exists; ' 'replay will load and append entries to archive file', options.cache_miss_file) cache_misses = cachemissarchive.CacheMissArchive.Load( options.cache_miss_file) else: cache_misses = cachemissarchive.CacheMissArchive( options.cache_miss_file) if options.server: AddDnsForward(server_manager, platform_settings, options.server) else: host = platform_settings.get_server_ip_address(options.server_mode) real_dns_lookup = dnsproxy.RealDnsLookup( name_servers=[platform_settings.get_original_primary_dns()]) if options.record: http_archive = httparchive.HttpArchive() http_archive.AssertWritable(replay_filename) else: http_archive = httparchive.HttpArchive.Load(replay_filename) logging.info('Loaded %d responses from %s', len(http_archive), replay_filename) server_manager.AppendRecordCallback(real_dns_lookup.ClearCache) server_manager.AppendRecordCallback(http_archive.clear) if options.dns_forwarding: if not options.server_mode: AddDnsForward(server_manager, platform_settings, host) AddDnsProxy(server_manager, options, host, real_dns_lookup, http_archive) if options.ssl and options.certfile is None: options.certfile = platform_settings.get_certfile_name() server_manager.AppendStartStopFunctions( [platform_settings.create_certfile, options.certfile], [os.unlink, options.certfile]) AddWebProxy(server_manager, options, host, real_dns_lookup, http_archive, cache_misses) AddTrafficShaper(server_manager, options, host) exit_status = 0 try: server_manager.Run() except KeyboardInterrupt: logging.info('Shutting down.') except (dnsproxy.DnsProxyException, trafficshaper.TrafficShaperException, platformsettings.NotAdministratorError, platformsettings.DnsUpdateError) as e: logging.critical('%s: %s', e.__class__.__name__, e) exit_status = 1 except: logging.critical(traceback.format_exc()) exit_status = 2 if options.record: http_archive.Persist(replay_filename) logging.info('Saved %d responses to %s', len(http_archive), replay_filename) if cache_misses: cache_misses.Persist() logging.info('Saved %d cache misses and %d requests to %s', cache_misses.get_total_cache_misses(), len(cache_misses.request_counts.keys()), options.cache_miss_file) return exit_status
def test_get_total_referers_small(self): cache_archive = cachemissarchive.CacheMissArchive('empty-archive') self.assertEqual(cache_archive.get_total_referers(), 0) referer = 'mock_referer' cache_archive.record_cache_miss(self.REQUEST, page_url=referer) self.assertEqual(cache_archive.get_total_referers(), 1)
def test_record_cache_miss(self): cache_archive = cachemissarchive.CacheMissArchive('empty-archive') referer = 'mock_referer' cache_archive.record_cache_miss(self.REQUEST, page_url=referer) self.assert_(cache_archive.archive[referer])
def test_init(self): empty_archive = cachemissarchive.CacheMissArchive('empty-archive') self.assert_(not empty_archive.archive)