def test_TimingSplit(self):
# Timing adds node 1 as a parent to 2 but not 3.
requests = [
self.MakeParserRequest(0,
'null',
100,
110,
magic_content_type=True),
self.MakeParserRequest(1, 0, 115, 120, magic_content_type=True),
self.MakeParserRequest(2, 0, 121, 122, magic_content_type=True),
self.MakeParserRequest(3, 0, 112, 119),
self.MakeParserRequest(4, 2, 122, 126),
self.MakeParserRequest(5, 2, 122, 126)
]
graph = loading_model.ResourceGraph(requests)
self.assertEqual(self.SuccessorIndicies(graph._nodes[0]), [1, 3])
self.assertEqual(self.SuccessorIndicies(graph._nodes[1]), [2])
self.assertEqual(self.SuccessorIndicies(graph._nodes[2]), [4, 5])
self.assertEqual(self.SuccessorIndicies(graph._nodes[3]), [])
self.assertEqual(self.SuccessorIndicies(graph._nodes[4]), [])
self.assertEqual(self.SuccessorIndicies(graph._nodes[5]), [])
self.assertEqual(self.SortedIndicies(graph), [0, 1, 3, 2, 4, 5])
# Change node 1 so it is a parent of 3, which become parent of 2.
requests[1] = self.MakeParserRequest(1,
0,
110,
111,
magic_content_type=True)
graph = loading_model.ResourceGraph(requests)
self.assertEqual(self.SuccessorIndicies(graph._nodes[0]), [1])
self.assertEqual(self.SuccessorIndicies(graph._nodes[1]), [3])
self.assertEqual(self.SuccessorIndicies(graph._nodes[2]), [4, 5])
self.assertEqual(self.SuccessorIndicies(graph._nodes[3]), [2])
self.assertEqual(self.SuccessorIndicies(graph._nodes[4]), [])
self.assertEqual(self.SuccessorIndicies(graph._nodes[5]), [])
self.assertEqual(self.SortedIndicies(graph), [0, 1, 3, 2, 4, 5])
# Add an initiator dependence to 1 that will become the parent of 3.
requests[1] = self.MakeParserRequest(1, 0, 110, 111)
requests.append(self.MakeParserRequest(6, 1, 111, 112))
graph = loading_model.ResourceGraph(requests)
# Check it doesn't change until we change the content type of 1.
self.assertEqual(self.SuccessorIndicies(graph._nodes[1]), [3, 6])
requests[1] = self.MakeParserRequest(1,
0,
110,
111,
magic_content_type=True)
graph = loading_model.ResourceGraph(requests)
self.assertEqual(self.SuccessorIndicies(graph._nodes[0]), [1])
self.assertEqual(self.SuccessorIndicies(graph._nodes[1]), [6])
self.assertEqual(self.SuccessorIndicies(graph._nodes[2]), [4, 5])
self.assertEqual(self.SuccessorIndicies(graph._nodes[3]), [2])
self.assertEqual(self.SuccessorIndicies(graph._nodes[4]), [])
self.assertEqual(self.SuccessorIndicies(graph._nodes[5]), [])
self.assertEqual(self.SuccessorIndicies(graph._nodes[6]), [3])
self.assertEqual(self.SortedIndicies(graph), [0, 1, 6, 3, 2, 4, 5])
def _FullFetch(url, json_output, prefetch):
"""Do a full fetch with optional prefetching."""
if not url.startswith('http') and not url.startswith('file'):
url = 'http://' + url
logging.warning('Cold fetch')
cold_data = _LogRequests(url)
assert cold_data, 'Cold fetch failed to produce data. Check your phone.'
if prefetch:
assert not OPTIONS.local
logging.warning('Generating prefetch')
prefetch_html = _GetPrefetchHtml(
loading_model.ResourceGraph(cold_data), name=url)
tmp = tempfile.NamedTemporaryFile()
tmp.write(prefetch_html)
tmp.flush()
# We hope that the tmpfile name is unique enough for the device.
target = os.path.join('/sdcard/Download', os.path.basename(tmp.name))
device = device_setup.GetFirstDevice()
device.adb.Push(tmp.name, target)
logging.warning('Pushed prefetch %s to device at %s' %
(tmp.name, target))
_LoadPage(device, 'file://' + target)
time.sleep(OPTIONS.prefetch_delay_seconds)
logging.warning('Warm fetch')
warm_data = _LogRequests(url, clear_cache_override=False)
with open(json_output, 'w') as f:
_WriteJson(f, warm_data)
logging.warning('Wrote ' + json_output)
with open(json_output + '.cold', 'w') as f:
_WriteJson(f, cold_data)
logging.warning('Wrote ' + json_output + '.cold')
else:
with open(json_output, 'w') as f:
_WriteJson(f, cold_data)
logging.warning('Wrote ' + json_output)
def test_EndToEnd(self):
# Test that we don't crash. This also runs through frame_load_lens.
tmp = tempfile.NamedTemporaryFile()
with gzip.GzipFile(self._ROLLING_STONE) as f:
trace = loading_trace.LoadingTrace.FromJsonDict(json.load(f))
frame_lens = frame_load_lens.FrameLoadLens(trace)
graph = loading_model.ResourceGraph(trace=trace, frame_lens=frame_lens)
visualization = model_graph.GraphVisualization(graph)
visualization.OutputDot(tmp)
def _ProcessRequests(filename):
with open(filename) as f:
trace = loading_trace.LoadingTrace.FromJsonDict(json.load(f))
content_lens = (content_classification_lens.ContentClassificationLens.
WithRulesFiles(trace, OPTIONS.ad_rules,
OPTIONS.tracking_rules))
frame_lens = frame_load_lens.FrameLoadLens(trace)
activity = activity_lens.ActivityLens(trace)
graph = loading_model.ResourceGraph(trace, content_lens, frame_lens,
activity)
if OPTIONS.noads:
graph.Set(node_filter=graph.FilterAds)
return graph
def test_MaxPath(self):
requests = [
self.MakeParserRequest(0, 'null', 100, 110),
self.MakeParserRequest(1, 0, 115, 120),
self.MakeParserRequest(2, 0, 112, 120),
self.MakeParserRequest(3, 1, 122, 126),
self.MakeParserRequest(4, 3, 127, 128),
self.MakeParserRequest(5, 'null', 100, 105),
self.MakeParserRequest(6, 5, 105, 110)
]
graph = loading_model.ResourceGraph(requests)
path_list = []
self.assertEqual(28, graph.Cost(path_list))
self.assertEqual([0, 1, 3, 4], [n.Index() for n in path_list])
def ColdGraph(datadir, site):
"""Return a loading model graph for the cold pull of site.
Based on ./analyze.py fetch file name conventions.
Args:
datadir: the directory containing site JSON data.
site: a site string.
Returns:
A loading model object.
"""
with file(os.path.join(datadir, site + '.json.cold')) as f:
return loading_model.ResourceGraph(loading_trace.LoadingTrace.FromJsonDict(
json.load(f)))
def WarmGraph(datadir, site):
"""Return a loading model graph for the warm pull of site.
Based on ./analyze.py fetch file name conventions.
Args:
datadir: the directory containing site JSON data.
site: a site string.
Returns:
A loading model object.
"""
return loading_model.ResourceGraph(
log_parser.FilterRequests(
log_parser.ParseJsonFile(os.path.join(datadir, site + '.json'))))
def _Main():
import json
import logging
import sys
import loading_model
import loading_trace
import resource_sack
sack = resource_sack.GraphSack()
for fname in sys.argv[1:]:
trace = loading_trace.LoadingTrace.FromJsonDict(
json.load(open(fname)))
logging.info('Making graph from %s', fname)
model = loading_model.ResourceGraph(trace, content_lens=None)
sack.ConsumeGraph(model)
logging.info('Finished %s', fname)
ToDot(sack, sys.stdout, prune=.1)
def test_TimingSplitImage(self):
# If we're all image types, then we shouldn't split by timing.
requests = [
self.MakeParserRequest(0, 'null', 100, 110),
self.MakeParserRequest(1, 0, 115, 120),
self.MakeParserRequest(2, 0, 121, 122),
self.MakeParserRequest(3, 0, 112, 119),
self.MakeParserRequest(4, 2, 122, 126),
self.MakeParserRequest(5, 2, 122, 126)
]
for r in requests:
r.headers['Content-Type'] = 'image/gif'
graph = loading_model.ResourceGraph(requests)
self.assertEqual(self.SuccessorIndicies(graph._nodes[0]), [1, 2, 3])
self.assertEqual(self.SuccessorIndicies(graph._nodes[1]), [])
self.assertEqual(self.SuccessorIndicies(graph._nodes[2]), [4, 5])
self.assertEqual(self.SuccessorIndicies(graph._nodes[3]), [])
self.assertEqual(self.SuccessorIndicies(graph._nodes[4]), [])
self.assertEqual(self.SuccessorIndicies(graph._nodes[5]), [])
self.assertEqual(self.SortedIndicies(graph), [0, 1, 2, 3, 4, 5])
def test_Costing(self):
requests = [
self.MakeParserRequest(0, 'null', 100, 110),
self.MakeParserRequest(1, 0, 115, 120),
self.MakeParserRequest(2, 0, 112, 120),
self.MakeParserRequest(3, 1, 122, 126),
self.MakeParserRequest(4, 3, 127, 128),
self.MakeParserRequest(5, 'null', 100, 105),
self.MakeParserRequest(6, 5, 105, 110)
]
graph = loading_model.ResourceGraph(requests)
self.assertEqual(self.SuccessorIndicies(graph._nodes[0]), [1, 2])
self.assertEqual(self.SuccessorIndicies(graph._nodes[1]), [3])
self.assertEqual(self.SuccessorIndicies(graph._nodes[2]), [])
self.assertEqual(self.SuccessorIndicies(graph._nodes[3]), [4])
self.assertEqual(self.SuccessorIndicies(graph._nodes[4]), [])
self.assertEqual(self.SuccessorIndicies(graph._nodes[5]), [6])
self.assertEqual(self.SuccessorIndicies(graph._nodes[6]), [])
self.assertEqual(self.SortedIndicies(graph), [0, 5, 1, 2, 6, 3, 4])
self.assertEqual(28, graph.Cost())
graph.Set(cache_all=True)
self.assertEqual(8, graph.Cost())
def testUserSatisfiedLens(self):
# We track all times in milliseconds, but raw trace events are in
# microseconds.
MILLI_TO_MICRO = 1000
loading_trace = test_utils.LoadingTraceFromEvents(
[self._RequestAt(1),
self._RequestAt(10),
self._RequestAt(20)],
trace_events=[{
'ts': 0,
'ph': 'I',
'cat': 'blink.some_other_user_timing',
'name': 'firstContentfulPaint'
}, {
'ts': 9 * MILLI_TO_MICRO,
'ph': 'I',
'cat': 'blink.user_timing',
'name': 'firstDiscontentPaint'
}, {
'ts': 12 * MILLI_TO_MICRO,
'ph': 'I',
'cat': 'blink.user_timing',
'name': 'firstContentfulPaint'
}, {
'ts': 22 * MILLI_TO_MICRO,
'ph': 'I',
'cat': 'blink.user_timing',
'name': 'firstContentfulPaint'
}])
graph = loading_model.ResourceGraph(loading_trace)
lens = user_satisfied_lens.UserSatisfiedLens(loading_trace, graph)
for n in graph.Nodes():
if n.Request().frame_id == '123.20':
self.assertFalse(lens.Filter(n))
else:
self.assertTrue(lens.Filter(n))
def _ProcessJson(json_data):
assert json_data
return loading_model.ResourceGraph(log_parser.FilterRequests(
[log_parser.RequestData.FromDict(r) for r in json_data]))
def _ProcessRequests(filename): requests = log_parser.FilterRequests(log_parser.ParseJsonFile(filename)) return loading_model.ResourceGraph(requests)