def test_matches(self): headers = {} request1 = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/index.html?hello=world', None, headers) request2 = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/index.html?foo=bar', None, headers) self.assert_(not request1.matches( request2.command, request2.host, request2.path, use_query=True)) self.assert_( request1.matches(request2.command, request2.host, request2.path, use_query=False)) self.assert_( request1.matches(request2.command, request2.host, None, use_query=True)) self.assert_( request1.matches(request2.command, None, request2.path, use_query=False)) empty_request = httparchive.ArchivedHttpRequest( None, None, None, None, headers) self.assert_(not empty_request.matches( request2.command, request2.host, None, use_query=True)) self.assert_(not empty_request.matches( request2.command, None, request2.path, use_query=False))
def setup_find_closest_request(self): headers = {} request1 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/a?hello=world', None, headers) request2 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/a?foo=bar', None, headers) request3 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/b?hello=world', None, headers) archive = httparchive.HttpArchive() # Add requests 2 and 3 and find closest match with request1 archive[request2] = self.RESPONSE archive[request3] = self.RESPONSE return archive, request1, request2, request3
def test_get_unmodified_headers(self): request = self.REQUEST response = self.RESPONSE archive = self.archive not_modified_response = httparchive.create_response(304) # Succeed check request_headers = {'if-unmodified-since': self.DATE_PAST} request = create_request(request_headers) self.assertEqual(archive.get(request), not_modified_response) # Fail check request_headers = {'if-unmodified-since': self.DATE_FUTURE} request = create_request(request_headers) self.assertEqual(archive.get(request), response) # Succeed check request_headers = {'if-unmodified-since': self.DATE_PRESENT} request = create_request(request_headers) self.assertEqual(archive.get(request), not_modified_response) # Fail check request_headers = {'if-unmodified-since': self.DATE_INVALID} request = create_request(request_headers) self.assertEqual(archive.get(request), response) # Fail check since the request is not a GET or HEAD request (as per RFC) request_headers = {'if-modified-since': self.DATE_PAST} request = httparchive.ArchivedHttpRequest('POST', 'www.test.com', '/', None, request_headers) self.assertEqual(archive.get(request), response)
def setUp(self): self.archive = httparchive.HttpArchive() self.archive[self.REQUEST] = self.RESPONSE # Also add an identical POST request for testing request = httparchive.ArchivedHttpRequest('POST', 'www.test.com', '/', None, self.REQUEST_HEADERS) self.archive[request] = self.RESPONSE
def test_get_cmp_seq(self): # The order of key-value pairs in query and header respectively should not # matter. headers = {'k2': 'v2', 'k1': 'v1'} request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/a?c=d&a=b;e=f', None, headers) self.assertEqual([('a', 'b'), ('c', 'd'), ('e', 'f'), ('k1', 'v1'), ('k2', 'v2')], request._GetCmpSeq('c=d&a=b;e=f'))
def testFetch(self): request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/', None, {}) response = self.createTestResponse() archive = httparchive.HttpArchive() archive[request] = response fetch = httpclient.ReplayHttpArchiveFetch(archive, None, self.dummy_injector) self.checkTestResponse(fetch(request), archive, request)
def testFetch(self, real_http_fetch): http_fetch_instance = real_http_fetch.return_value response = self.createTestResponse() http_fetch_instance.return_value = response archive = httparchive.HttpArchive() fetch = httpclient.RecordHttpArchiveFetch(archive, self.dummy_injector) request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/', None, {}) self.checkTestResponse(fetch(request), archive, request)
def testFetchSSLRequest(self): real_dns_lookup = dnsproxy.RealDnsLookup( name_servers=[platformsettings.get_original_primary_nameserver()]) fetch = httpclient.RealHttpFetch(real_dns_lookup) request = httparchive.ArchivedHttpRequest( command='GET', host='google.com', full_path='/search?q=dogs', request_body=None, headers={}, is_ssl=True) response = fetch(request) self.assertIsNotNone(response)
def test_find_closest_request_timestamp(self): headers = {} request1 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/index.html?time=100000000&important=true', None, headers) request2 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/index.html?time=99999999&important=true', None, headers) request3 = httparchive.ArchivedHttpRequest( 'GET', 'www.test.com', '/index.html?time=10000000&important=false', None, headers) archive = httparchive.HttpArchive() # Add requests 2 and 3 and find closest match with request1 archive[request2] = self.RESPONSE archive[request3] = self.RESPONSE # Although request3 is lexicographically closer, request2 is semantically # more similar. self.assertEqual(request2, archive.find_closest_request(request1, use_path=True))
def request_handler(self, method, uri, hdrs, res_start, req_pause): """ Based on method, host and uri to fetch the matching response and reply to browser using spdy. """ dummy = http_common.dummy def simple_responder(code, content): res_hdrs = [('content-type', 'text/html'), ('version', 'HTTP/1.1')] res_body, res_done = res_start(str(code), content, res_hdrs, dummy) res_body(None) res_done(None) host = '' for name, value in hdrs: if name.lower() == 'host': host = value self.log.debug("request: %s, uri: %s, method: %s", host, uri, method) if method == 'GET': request = httparchive.ArchivedHttpRequest(method, host, uri, None, dict(hdrs)) response_code = self.custom_handlers.handle(request) if response_code: simple_responder(response_code, "Handled by custom handlers") return dummy, dummy response = self.http_archive_fetch(request) if response: res_hdrs = [('version', 'HTTP/1.1')] for name, value in response.headers: name_lower = name.lower() if name_lower == CONTENT_LENGTH: res_hdrs.append((name, str(value))) elif name_lower in (STATUS, VERSION): pass else: res_hdrs.append((name_lower, value)) res_body, res_done = res_start(str(response.status), response.reason, res_hdrs, dummy) body = '' for item in response.response_data: res_body(item) res_done(None) else: self.log.error("404 returned: %s %s", method, uri) simple_responder(404, "file not found") else: # TODO(lzheng): Add support for other methods. self.log.error("method: %s is not supported: %s", method, uri) simple_responder(500, "Not supported") return dummy, dummy
def get_archived_http_request(self): host = self.headers.get('host') if host is None: logging.error('Request without host header') self.send_error(500) return None parsed = urlparse.urlparse(self.path) params = ';%s' % parsed.params if parsed.params else '' query = '?%s' % parsed.query if parsed.query else '' fragment = '#%s' % parsed.fragment if parsed.fragment else '' full_path = '%s%s%s%s' % (parsed.path, params, query, fragment) repr_path = full_path # remove all designated groups from the matched URL. # e.g if the pattern is "(.*\.)?foo.com/bar.*(qux=1&).*" # then "abc.foo.com/bart?qux=1&z" --> "foo.com/bart?z" for path in self.server.paths_to_edit: groups = path.match('%s%s' % (host, repr_path)) if groups: start_include = 0 new_path = '' for i in xrange(groups.lastindex): start_group, end_group = groups.span(i + 1) new_path += repr_path[start_include:start_group] start_include = end_group repr_path = new_path + repr_path[start_include::] logging.info( 'While %sing replaced %s with %s', ('record' if self.server.http_archive_fetch.is_record_mode else 'replay'), full_path, repr_path) exclude_headers = [] for path, undesirable_key in self.server.undesirable_headers.items(): if re.match(r'%s' % path, parsed.path): exclude_headers.append(undesirable_key) for path, status in self.server.error_paths: if path.match('%s%s' % (host, full_path)): logging.debug('Send %d for %s%s', status, host, full_path) logging.debug(path.pattern) self.send_error(status) return None return httparchive.ArchivedHttpRequest(self.command, host, full_path, self.read_request_body(), self.get_header_dict(), self.server.is_ssl, repr_path, exclude_headers)
def convert_request(request, is_ssl): command = convert_unicode(request["method"]) url = convert_unicode(request["url"]) url_object = urlparse.urlsplit(url) host = url_object.netloc # TODO(cs): must be a more robust way to get the full path. urlparse # doesn't seem to have it though. full_path = url[url.find(host) + len(host):] request_body = None if "postData" in request: request_body = convert_unicode(request["postData"]) headers = convert_headers_to_dict(request["headers"]) return httparchive.ArchivedHttpRequest(command, host, full_path, request_body, headers, is_ssl=is_ssl)
def get_archived_http_request(self): host = self.headers.get('host') if host is None: logging.error('Request without host header') return None parsed = urlparse.urlparse(self.path) query = '?%s' % parsed.query if parsed.query else '' fragment = '#%s' % parsed.fragment if parsed.fragment else '' full_path = '%s%s%s' % (parsed.path, query, fragment) return httparchive.ArchivedHttpRequest(self.command, host, full_path, self.read_request_body(), self.get_header_dict(), self.server.is_ssl)
def testInjectedDate(self, os_path, util_exists, util_resource_string): os_path.return_value = False util_exists.return_value = True util_resource_string.return_value = \ ["""var time_seed={}""".format(script_injector.TIME_SEED_MARKER)] request = httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/', None, {}) response = self.createTestResponse() archive = httparchive.HttpArchive() archive[request] = response fetch = httpclient.ReplayHttpArchiveFetch( archive, None, script_injector.GetScriptInjector("time_script.js")) self.assertEqual( ['<script>var time_seed=1479344523000</script><body>test</body>'], fetch(request).response_data)
def get_archived_http_request(self): host = self.headers.get('host') if host is None: logging.error('Request without host header') return None parsed = urlparse.urlparse(self.path) params = ';%s' % parsed.params if parsed.params else '' query = '?%s' % parsed.query if parsed.query else '' fragment = '#%s' % parsed.fragment if parsed.fragment else '' full_path = '%s%s%s%s' % (parsed.path, params, query, fragment) StubRequest = collections.namedtuple('StubRequest', ('host', 'full_path')) request, response = StubRequest(host, full_path), None self.server.log_url(request, response) return httparchive.ArchivedHttpRequest(self.command, host, full_path, self.read_request_body(), self.get_header_dict(), self.server.is_ssl)
def create_request(headers): return httparchive.ArchivedHttpRequest('GET', 'www.test.com', '/', None, headers)