def testAsyncScriptDependency(self):
JS_REQUEST_WITH_ASYNC_STACK = Request.FromJsonDict({
'url':
'http://bla.com/cat.js',
'request_id':
'1234.14',
'initiator': {
'type': 'script',
'stack': {
'callFrames': [],
'parent': {
'callFrames': [{
'url': 'http://bla.com/nyancat.js'
}]
}
}
},
'timestamp':
10,
'timing':
TimingFromDict({})
})
loading_trace = test_utils.LoadingTraceFromEvents(
[TestRequests.JS_REQUEST, JS_REQUEST_WITH_ASYNC_STACK])
request_dependencies_lens = RequestDependencyLens(loading_trace)
deps = request_dependencies_lens.GetRequestDependencies()
self.assertEquals(1, len(deps))
self._AssertDependencyIs(deps[0], TestRequests.JS_REQUEST.request_id,
JS_REQUEST_WITH_ASYNC_STACK.request_id,
'script')
def testParserDependency(self):
loading_trace = test_utils.LoadingTraceFromEvents(
[TestRequests.REQUEST, TestRequests.JS_REQUEST])
request_dependencies_lens = RequestDependencyLens(loading_trace)
deps = request_dependencies_lens.GetRequestDependencies()
self.assertEquals(1, len(deps))
self._AssertDependencyIs(deps[0], TestRequests.REQUEST.request_id,
TestRequests.JS_REQUEST.request_id, 'parser')
def testScriptDependency(self):
loading_trace = test_utils.LoadingTraceFromEvents(
[self._JS_REQUEST, self._JS_REQUEST_2])
request_dependencies_lens = RequestDependencyLens(loading_trace)
deps = request_dependencies_lens.GetRequestDependencies()
self.assertEquals(1, len(deps))
self._AssertDependencyIs(deps[0], self._JS_REQUEST.request_id,
self._JS_REQUEST_2.request_id, 'script')
def testDependencyDifferentFrame(self):
"""Checks that a more recent request from another frame is ignored."""
loading_trace = test_utils.LoadingTraceFromEvents([
self._JS_REQUEST, self._JS_REQUEST_OTHER_FRAME, self._JS_REQUEST_2
])
request_dependencies_lens = RequestDependencyLens(loading_trace)
deps = request_dependencies_lens.GetRequestDependencies()
self.assertEquals(1, len(deps))
self._AssertDependencyIs(deps[0], self._JS_REQUEST.request_id,
self._JS_REQUEST_2.request_id, 'script')
def testRedirectDependency(self):
loading_trace = test_utils.LoadingTraceFromEvents(
[self._REDIRECT_REQUEST, self._REDIRECTED_REQUEST])
request_dependencies_lens = RequestDependencyLens(loading_trace)
deps = request_dependencies_lens.GetRequestDependencies()
self.assertEquals(1, len(deps))
(first, second, reason) = deps[0]
self.assertEquals('redirect', reason)
self.assertEquals(self._REDIRECT_REQUEST.request_id, first.request_id)
self.assertEquals(self._REQUEST.request_id, second.request_id)
def testDependencySameParentFrame(self):
"""Checks that a more recent request from an unrelated frame is ignored
if there is one from a related frame."""
loading_trace = test_utils.LoadingTraceFromEvents([
TestRequests.JS_REQUEST_OTHER_FRAME,
TestRequests.JS_REQUEST_UNRELATED_FRAME, TestRequests.JS_REQUEST_2
], TestRequests.PAGE_EVENTS)
request_dependencies_lens = RequestDependencyLens(loading_trace)
deps = request_dependencies_lens.GetRequestDependencies()
self.assertEquals(1, len(deps))
self._AssertDependencyIs(
deps[0], TestRequests.JS_REQUEST_OTHER_FRAME.request_id,
TestRequests.JS_REQUEST_2.request_id, 'script')
def testSeveralDependencies(self):
loading_trace = test_utils.LoadingTraceFromEvents([
self._REDIRECT_REQUEST, self._REDIRECTED_REQUEST, self._JS_REQUEST,
self._JS_REQUEST_2
])
request_dependencies_lens = RequestDependencyLens(loading_trace)
deps = request_dependencies_lens.GetRequestDependencies()
self.assertEquals(3, len(deps))
self._AssertDependencyIs(deps[0], self._REDIRECT_REQUEST.request_id,
self._REQUEST.request_id, 'redirect')
self._AssertDependencyIs(deps[1], self._REQUEST.request_id,
self._JS_REQUEST.request_id, 'parser')
self._AssertDependencyIs(deps[2], self._JS_REQUEST.request_id,
self._JS_REQUEST_2.request_id, 'script')
def testGetRedirectChain(self):
loading_trace = test_utils.LoadingTraceFromEvents(
[TestRequests.FIRST_REDIRECT_REQUEST,
TestRequests.SECOND_REDIRECT_REQUEST, TestRequests.REDIRECTED_REQUEST])
request_dependencies_lens = RequestDependencyLens(loading_trace)
whole_chain = [TestRequests.FIRST_REDIRECT_REQUEST,
TestRequests.SECOND_REDIRECT_REQUEST,
TestRequests.REDIRECTED_REQUEST]
chain = request_dependencies_lens.GetRedirectChain(
TestRequests.FIRST_REDIRECT_REQUEST)
self.assertListEqual(whole_chain, chain)
chain = request_dependencies_lens.GetRedirectChain(
TestRequests.SECOND_REDIRECT_REQUEST)
self.assertListEqual(whole_chain[1:], chain)
chain = request_dependencies_lens.GetRedirectChain(
TestRequests.REDIRECTED_REQUEST)
self.assertEquals(whole_chain[2:], chain)
def ExtractDiscoverableUrls(loading_trace_path, subresource_discoverer):
"""Extracts discoverable resource urls from a loading trace according to a
sub-resource discoverer.
Args:
loading_trace_path: The loading trace's path.
subresource_discoverer: The sub-resources discoverer that should white-list
the resources to keep in cache for the NoState-Prefetch benchmarks.
Returns:
A set of urls.
"""
assert subresource_discoverer in SUBRESOURCE_DISCOVERERS, \
'unknown prefetch simulation {}'.format(subresource_discoverer)
# Load trace and related infos.
logging.info('loading %s' % loading_trace_path)
trace = LoadingTrace.FromJsonFile(loading_trace_path)
dependencies_lens = RequestDependencyLens(trace)
first_resource_request = trace.request_track.GetFirstResourceRequest()
# Build the list of discovered requests according to the desired simulation.
discovered_requests = []
if subresource_discoverer == EMPTY_CACHE_DISCOVERER:
pass
elif subresource_discoverer == FULL_CACHE_DISCOVERER:
discovered_requests = trace.request_track.GetEvents()
elif subresource_discoverer == REDIRECTED_MAIN_DISCOVERER:
discovered_requests = \
[dependencies_lens.GetRedirectChain(first_resource_request)[-1]]
elif subresource_discoverer == PARSER_DISCOVERER:
discovered_requests = PrefetchSimulationView.ParserDiscoverableRequests(
first_resource_request, dependencies_lens)
elif subresource_discoverer == HTML_PRELOAD_SCANNER_DISCOVERER:
discovered_requests = PrefetchSimulationView.PreloadedRequests(
first_resource_request, dependencies_lens, trace)
else:
assert False
whitelisted_urls = set()
logging.info('white-listing %s' % first_resource_request.url)
for request in _FilterOutDataAndIncompleteRequests(discovered_requests):
logging.info('white-listing %s' % request.url)
whitelisted_urls.add(request.url)
return whitelisted_urls
def _ExtractDiscoverableUrls(original_headers_path, loading_trace_path,
subresource_discoverer):
"""Extracts discoverable resource urls from a loading trace according to a
sub-resource discoverer.
Args:
original_headers_path: Path of JSON containing the original headers.
loading_trace_path: Path of the loading trace recorded at original cache
creation.
subresource_discoverer: The sub-resources discoverer that should white-list
the resources to keep in cache for the NoState-Prefetch benchmarks.
Returns:
A set of urls.
"""
assert subresource_discoverer in SUBRESOURCE_DISCOVERERS, \
'unknown prefetch simulation {}'.format(subresource_discoverer)
logging.info('loading %s', loading_trace_path)
trace = loading_trace.LoadingTrace.FromJsonFile(loading_trace_path)
dependencies_lens = RequestDependencyLens(trace)
# Build the list of discovered requests according to the desired simulation.
discovered_requests = []
if subresource_discoverer == Discoverer.HTMLPreloadScannerStore:
requests = _DiscoverRequests(dependencies_lens,
Discoverer.HTMLPreloadScanner)
discovered_requests = _PruneOutOriginalNoStoreRequests(
original_headers_path, requests)
else:
discovered_requests = _DiscoverRequests(dependencies_lens,
subresource_discoverer)
whitelisted_urls = set()
for request in sandwich_utils.FilterOutDataAndIncompleteRequests(
discovered_requests):
logging.debug('white-listing %s', request.url)
whitelisted_urls.add(request.url)
logging.info('number of white-listed resources: %d', len(whitelisted_urls))
return whitelisted_urls