def setUp(self): self.archive = httparchive.HttpArchive() self.archive[self.REQUEST] = self.RESPONSE # Also add an identical POST request for testing request = httparchive.ArchivedHttpRequest('POST', 'www.test.com', '/', None, self.REQUEST_HEADERS) self.archive[request] = self.RESPONSE
def testFetch(self): request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/', None, {}) response = self.createTestResponse() archive = httparchive.HttpArchive() archive[request] = response fetch = httpclient.ReplayHttpArchiveFetch(archive, None, self.dummy_injector) self.checkTestResponse(fetch(request), archive, request)
def testFetch(self, real_http_fetch): http_fetch_instance = real_http_fetch.return_value response = self.createTestResponse() http_fetch_instance.return_value = response archive = httparchive.HttpArchive() fetch = httpclient.RecordHttpArchiveFetch(archive, self.dummy_injector) request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/', None, {}) self.checkTestResponse(fetch(request), archive, request)
def setup_find_closest_request(self): headers = {} request1 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/a?hello=world', None, headers) request2 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/a?foo=bar', None, headers) request3 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/b?hello=world', None, headers) archive = httparchive.HttpArchive() # Add requests 2 and 3 and find closest match with request1 archive[request2] = self.RESPONSE archive[request3] = self.RESPONSE return archive, request1, request2, request3
def testInjectedDate(self, os_path, util_exists, util_resource_string): os_path.return_value = False util_exists.return_value = True util_resource_string.return_value = \ ["""var time_seed={}""".format(script_injector.TIME_SEED_MARKER)] request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/', None, {}) response = self.createTestResponse() archive = httparchive.HttpArchive() archive[request] = response fetch = httpclient.ReplayHttpArchiveFetch( archive, None, script_injector.GetScriptInjector("time_script.js")) self.assertEqual( ['<script>var time_seed=1479344523000</script><body>test</body>'], fetch(request).response_data)
def convert_to_wpr(har): # Assumes one page load per har file. try: archive = httparchive.HttpArchive() for entry in har["log"]["entries"]: timings = entry["timings"] if is_favicon(entry): # Favicons don't affect page load time. continue maybe_fill_in_image_data(entry) # TODO(cs): find a better way to infer ssl is_ssl = "ssl" in timings and timings["ssl"] != -1 request = convert_request(entry["request"], is_ssl) response = convert_response(entry["response"], timings) archive[request] = response return archive except KeyError as e: traceback.print_exc(e) raise ValueError("Malformed HAR. " + str(e))
def test_find_closest_request_timestamp(self): headers = {} request1 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/index.html?time=100000000&important=true', None, headers) request2 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/index.html?time=99999999&important=true', None, headers) request3 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/index.html?time=10000000&important=false', None, headers) archive = httparchive.HttpArchive() # Add requests 2 and 3 and find closest match with request1 archive[request2] = self.RESPONSE archive[request3] = self.RESPONSE # Although request3 is lexicographically closer, request2 is semantically # more similar. self.assertEqual(request2, archive.find_closest_request(request1, use_path=True))
def replay(options, replay_filename): if options.admin_check and options.IsRootRequired(): platformsettings.rerun_as_administrator() configure_logging(options.log_level, options.log_file) server_manager = servermanager.ServerManager(options.record) cache_misses = None if options.cache_miss_file: if os.path.exists(options.cache_miss_file): logging.warning( 'Cache Miss Archive file %s already exists; ' 'replay will load and append entries to archive file', options.cache_miss_file) cache_misses = cachemissarchive.CacheMissArchive.Load( options.cache_miss_file) else: cache_misses = cachemissarchive.CacheMissArchive( options.cache_miss_file) if options.server: AddDnsForward(server_manager, options.server) else: real_dns_lookup = dnsproxy.RealDnsLookup( name_servers=[platformsettings.get_original_primary_nameserver()]) if options.record: httparchive.HttpArchive.AssertWritable(replay_filename) if options.append and os.path.exists(replay_filename): http_archive = httparchive.HttpArchive.Load(replay_filename) logging.info('Appending to %s (loaded %d existing responses)', replay_filename, len(http_archive)) else: http_archive = httparchive.HttpArchive() else: http_archive = httparchive.HttpArchive.Load(replay_filename) logging.info('Loaded %d responses from %s', len(http_archive), replay_filename) server_manager.AppendRecordCallback(real_dns_lookup.ClearCache) server_manager.AppendRecordCallback(http_archive.clear) ipfw_dns_host = None if options.dns_forwarding or options.shaping_dummynet: # compute the ip/host used for the DNS server and traffic shaping ipfw_dns_host = options.host if not ipfw_dns_host: ipfw_dns_host = platformsettings.get_server_ip_address( options.server_mode) if options.dns_forwarding: if not options.server_mode and ipfw_dns_host == '127.0.0.1': AddDnsForward(server_manager, ipfw_dns_host) AddDnsProxy(server_manager, options, ipfw_dns_host, options.dns_port, real_dns_lookup, http_archive) if options.ssl and options.https_root_ca_cert_path is None: options.https_root_ca_cert_path = os.path.join( os.path.dirname(__file__), 'wpr_cert.pem') http_proxy_address = options.host if not http_proxy_address: http_proxy_address = platformsettings.get_httpproxy_ip_address( options.server_mode) AddWebProxy(server_manager, options, http_proxy_address, real_dns_lookup, http_archive, cache_misses) AddTrafficShaper(server_manager, options, ipfw_dns_host) exit_status = 0 try: server_manager.Run() except KeyboardInterrupt: logging.info('Shutting down.') except (dnsproxy.DnsProxyException, trafficshaper.TrafficShaperException, platformsettings.NotAdministratorError, platformsettings.DnsUpdateError) as e: logging.critical('%s: %s', e.__class__.__name__, e) exit_status = 1 except: logging.critical(traceback.format_exc()) exit_status = 2 if options.record: http_archive.Persist(replay_filename) logging.info('Saved %d responses to %s', len(http_archive), replay_filename) if cache_misses: cache_misses.Persist() logging.info('Saved %d cache misses and %d requests to %s', cache_misses.get_total_cache_misses(), len(cache_misses.request_counts.keys()), options.cache_miss_file) return exit_status
def test_init(self): archive = httparchive.HttpArchive() self.assertEqual(len(archive), 0)
def replay(options, replay_filename): if options.record and sys.version_info < (2, 7, 9): print ('Need Python 2.7.9 or greater for recording mode.\n' 'For instructions on how to upgrade Python on Ubuntu 14.04, see:\n' 'http://mbless.de/blog/2016/01/09/upgrade-to-python-2711-on-ubuntu-1404-lts.html\n') if options.admin_check and options.IsRootRequired(): platformsettings.rerun_as_administrator() configure_logging(options.log_level, options.log_file) server_manager = servermanager.ServerManager(options.record) if options.server: AddDnsForward(server_manager, options.server) else: if options.record: httparchive.HttpArchive.AssertWritable(replay_filename) if options.append and os.path.exists(replay_filename): http_archive = httparchive.HttpArchive.Load(replay_filename) logging.info('Appending to %s (loaded %d existing responses)', replay_filename, len(http_archive)) else: http_archive = httparchive.HttpArchive() else: http_archive = httparchive.HttpArchive.Load(replay_filename) logging.info('Loaded %d responses from %s', len(http_archive), replay_filename) server_manager.AppendRecordCallback(http_archive.clear) ipfw_dns_host = None if options.dns_forwarding or options.shaping_dummynet: # compute the ip/host used for the DNS server and traffic shaping ipfw_dns_host = options.host if not ipfw_dns_host: ipfw_dns_host = platformsettings.get_server_ip_address( options.server_mode) real_dns_lookup = dnsproxy.RealDnsLookup( name_servers=[platformsettings.get_original_primary_nameserver()], dns_forwarding=options.dns_forwarding, proxy_host=ipfw_dns_host, proxy_port=options.dns_port) server_manager.AppendRecordCallback(real_dns_lookup.ClearCache) if options.dns_forwarding: if not options.server_mode and ipfw_dns_host == '127.0.0.1': AddDnsForward(server_manager, ipfw_dns_host) AddDnsProxy(server_manager, options, ipfw_dns_host, options.dns_port, real_dns_lookup, http_archive) if options.ssl and options.https_root_ca_cert_path is None: options.https_root_ca_cert_path = os.path.join(os.path.dirname(__file__), 'wpr_cert.pem') http_proxy_address = options.host if not http_proxy_address: http_proxy_address = platformsettings.get_httpproxy_ip_address( options.server_mode) AddWebProxy(server_manager, options, http_proxy_address, real_dns_lookup, http_archive) AddTrafficShaper(server_manager, options, ipfw_dns_host) exit_status = 0 try: server_manager.Run() except KeyboardInterrupt: logging.info('Shutting down.') except (dnsproxy.DnsProxyException, trafficshaper.TrafficShaperException, platformsettings.NotAdministratorError, platformsettings.DnsUpdateError) as e: logging.critical('%s: %s', e.__class__.__name__, e) exit_status = 1 except Exception: logging.critical(traceback.format_exc()) exit_status = 2 if options.record: http_archive.Persist(replay_filename) logging.info('Saved %d responses to %s', len(http_archive), replay_filename) return exit_status
def replay(options, replay_filename): platform_settings = platformsettings.get_platform_settings() if options.IsRootRequired(): platform_settings.rerun_as_administrator() configure_logging(platform_settings, options.log_level, options.log_file) server_manager = servermanager.ServerManager(options.record) cache_misses = None if options.cache_miss_file: if os.path.exists(options.cache_miss_file): logging.warning('Cache Miss Archive file %s already exists; ' 'replay will load and append entries to archive file', options.cache_miss_file) cache_misses = cachemissarchive.CacheMissArchive.Load( options.cache_miss_file) else: cache_misses = cachemissarchive.CacheMissArchive( options.cache_miss_file) if options.server: AddDnsForward(server_manager, platform_settings, options.server) else: host = platform_settings.get_server_ip_address(options.server_mode) real_dns_lookup = dnsproxy.RealDnsLookup( name_servers=[platform_settings.get_original_primary_dns()]) if options.record: http_archive = httparchive.HttpArchive() http_archive.AssertWritable(replay_filename) else: http_archive = httparchive.HttpArchive.Load(replay_filename) logging.info('Loaded %d responses from %s', len(http_archive), replay_filename) server_manager.AppendRecordCallback(real_dns_lookup.ClearCache) server_manager.AppendRecordCallback(http_archive.clear) if options.dns_forwarding: if not options.server_mode: AddDnsForward(server_manager, platform_settings, host) AddDnsProxy(server_manager, options, host, real_dns_lookup, http_archive) if options.ssl and options.certfile is None: options.certfile = platform_settings.get_certfile_name() server_manager.AppendStartStopFunctions( [platform_settings.create_certfile, options.certfile], [os.unlink, options.certfile]) AddWebProxy(server_manager, options, host, real_dns_lookup, http_archive, cache_misses) AddTrafficShaper(server_manager, options, host) exit_status = 0 try: server_manager.Run() except KeyboardInterrupt: logging.info('Shutting down.') except (dnsproxy.DnsProxyException, trafficshaper.TrafficShaperException, platformsettings.NotAdministratorError, platformsettings.DnsUpdateError) as e: logging.critical('%s: %s', e.__class__.__name__, e) exit_status = 1 except: logging.critical(traceback.format_exc()) exit_status = 2 if options.record: http_archive.Persist(replay_filename) logging.info('Saved %d responses to %s', len(http_archive), replay_filename) if cache_misses: cache_misses.Persist() logging.info('Saved %d cache misses and %d requests to %s', cache_misses.get_total_cache_misses(), len(cache_misses.request_counts.keys()), options.cache_miss_file) return exit_status