def _AugmentFrameLoads(self, index_by_request):
if not self._frame_lens:
return
loads = self._frame_lens.GetFrameLoadInfo()
load_index_to_node = {}
for l in loads:
next_index = len(self._nodes)
node = dag.Node(next_index)
node_info = self._NodeInfo(node, (l.index, l.msec))
load_index_to_node[l.index] = next_index
self._nodes.append(node)
self._node_info.append(node_info)
frame_deps = self._frame_lens.GetFrameLoadDependencies()
for load_idx, rq in frame_deps[0]:
parent = self._node_info[load_index_to_node[load_idx]]
child = self._node_info[index_by_request[rq]]
parent.Node().AddSuccessor(child.Node())
parent.AddEdgeAnnotations(child,
{self.EDGE_KIND_KEY: 'after-load'})
for rq, load_idx in frame_deps[1]:
child = self._node_info[load_index_to_node[load_idx]]
parent = self._node_info[index_by_request[rq]]
parent.Node().AddSuccessor(child.Node())
parent.AddEdgeAnnotations(child,
{self.EDGE_KIND_KEY: 'before-load'})
def testfindLCA(self):
root = Node(1)
root.children.append(dag.Node(2))
root.children.append(dag.Node(3))
root.children[0].children.append(dag.Node(4))
root.children[0].children.append(dag.Node(5))
root.children[1].children.append(dag.Node(6))
root.children[1].children.append(dag.Node(7))
# test all nodes are None
self.assertEqual(dag.findLCA(None, None, None), None)
#test when root is euqal to node
self.assertEqual(dag.findLCA(root, 1, 1), 1)
#test if n1 is not present
self.assertEqual(dag.findLCA(root, None, 2), None)
#test if n2 is not present
self.assertEqual(dag.findLCA(root, 2, None), None)
#test when node is value is not in tree
self.assertEqual(dag.findLCA(root, 2, 13), None)
#test dag for different values
self.assertEqual(dag.findLCA(root, 3, 7), 3)
self.assertEqual(dag.findLCA(root, 4, 6), 1)
self.assertEqual(dag.findLCA(root, 2, 5), 2)
self.assertEqual(dag.findLCA(root, 3, 4), 1)
def test_find_lca_dag_input_error(self):
root = dag.Node(1)
r2 = dag.Node(2)
r3 = dag.Node(3)
r4 = dag.Node(4)
r5 = dag.Node(5)
r6 = dag.Node(6)
root.succ = [r2, r3]
r2.succ = [r5, r6]
r2.pred = [root]
r3.succ = [r4, r5]
r3.pred = [root]
r4.succ = [r5]
r4.pred = [r3]
r5.succ = [r6]
r5.pred = [r2, r3, r4]
r6.pred = [r2, r5]
lca_invalid_root = dag.find_lca_dag("a", r2, r3)
assert lca_invalid_root is None
lca_invalid_n1 = dag.find_lca_dag(root, "a", r3)
assert lca_invalid_n1 is None
lca_invalid_n2 = dag.find_lca_dag(root, r2, "a")
assert lca_invalid_n2 is None
def _BuildDag(self, trace):
"""Build DAG of resources.
Build a DAG from our requests and augment with NodeInfo (see above) in a
parallel array indexed by Node.Index().
Creates self._nodes and self._node_info.
Args:
trace: A LoadingTrace.
"""
self._nodes = []
self._node_info = []
index_by_request = {}
for request in trace.request_track.GetEvents():
next_index = len(self._nodes)
assert request not in index_by_request
index_by_request[request] = next_index
node = dag.Node(next_index)
node_info = self._NodeInfo(node, request)
if self._content_lens:
node_info.SetRequestContent(
self._content_lens.IsAdRequest(request),
self._content_lens.IsTrackingRequest(request))
self._nodes.append(node)
self._node_info.append(node_info)
dependencies = self.REQUEST_LENS(trace).GetRequestDependencies()
for dep in dependencies:
(parent_rq, child_rq, reason) = dep
parent = self._node_info[index_by_request[parent_rq]]
child = self._node_info[index_by_request[child_rq]]
edge_cost = request_track.TimeBetween(parent_rq, child_rq, reason)
if edge_cost < 0:
edge_cost = 0
if child.StartTime() < parent.StartTime():
logging.error('Inverted dependency: %s->%s',
parent.ShortName(), child.ShortName())
# Note that child.StartTime() < parent.EndTime() appears to happen a
# fair amount in practice.
parent.Node().AddSuccessor(child.Node())
parent.SetEdgeCost(child, edge_cost)
parent.AddEdgeAnnotations(child, {self.EDGE_KIND_KEY: reason})
if self._activity_lens:
activity = self._activity_lens.BreakdownEdgeActivityByInitiator(
dep)
parent.AddEdgeAnnotations(child, {'activity': activity})
self._AugmentFrameLoads(index_by_request)
def test_pathTo(self):
#Graph with 7 nodes
root = dag.Node(1)
root.children.append(dag.Node(2))
root.children.append(dag.Node(3))
root.children[0].children.append(dag.Node(4))
root.children[0].children.append(dag.Node(5))
root.children[1].children.append(dag.Node(6))
root.children[1].children[0].children.append(dag.Node(5))
root.children[1].children[0].children.append(dag.Node(7))
#Test 1: Test when root is None
self.assertEqual(dag.pathTo(None, 2, 3), False)
#Test 2: Test all values are None
self.assertEqual(dag.pathTo(None, None, None), False)
#Test 3: test path to root
self.assertEqual(dag.pathTo(root, [], 1), True)
#Test 4: test for key not in tree
self.assertEqual(dag.pathTo(root, [], 25), False)
#Test 4: path to 7
self.assertEqual(dag.pathTo(root, [], 7), True)
#Test 6: path to 6
self.assertEqual(dag.pathTo(root, [], 6), True)
#Test 7: path to 5
self.assertEqual(dag.pathTo(root, [], 5), True)
#Test 8: path to 4
self.assertEqual(dag.pathTo(root, [], 4), True)
#Test 9: path to 3
self.assertEqual(dag.pathTo(root, [], 3), True)
#Test 10: path to 2
self.assertEqual(dag.pathTo(root, [], 2), True)
def MakeDag(self, links):
"""Make a graph from a description of links.
Args:
links: A list of (index, (successor index...)) tuples. Index must equal
the location of the tuple in the list and are provided to make it easier
to read.
Returns:
A list of Nodes.
"""
nodes = []
for i in xrange(len(links)):
assert i == links[i][0]
nodes.append(dag.Node(i))
for l in links:
for s in l[1]:
nodes[l[0]].AddSuccessor(nodes[s])
return nodes
def test_findPathsTo(self):
#new Graph
root = dag.Node(1)
root.children.append(dag.Node(2))
root.children.append(dag.Node(3))
root.children[0].children.append(dag.Node(4))
root.children[1].children.append(dag.Node(6))
root.children[1].children[0].children.append(dag.Node(5))
root.children[1].children[0].children.append(dag.Node(7))
#Test 1: test when root is equal None
self.assertEqual(dag.findPathsTo(None, 2), [])
#Test 2: test when key is None
self.assertEqual(dag.findPathsTo(root, None), [])
#Test 3: test when all keys are None
self.assertEqual(dag.findPathsTo(None, None), [])
#Test 4: test path to root
self.assertEqual(dag.findPathsTo(root, 1), [[1]])
#Test 5: test for key not in tree
self.assertEqual(dag.findPathsTo(root, 25), [])
#Test 6: paths to 7
self.assertEqual(dag.findPathsTo(root, 7), [[1, 3, 6, 7]])
#Test 7: path to 5
self.assertEqual(dag.findPathsTo(root, 5), [[1, 3, 6, 5]])
#Test 8: path to 4
self.assertEqual(dag.findPathsTo(root, 4), [[1, 2, 4]])
#Test 9: test for more than one path
root.children[0].children[0].children.append(dag.Node(5))
self.assertEqual(dag.findPathsTo(root, 5),
[[1, 2, 4, 5], [1, 3, 6, 5]])
def test_findLCA_v2(self):
#Test 1: test all values equal None
self.assertEqual(dag.findLCA(None, None, None), None)
#Test 2: test when both node are root
root = dag.Node(1)
self.assertEqual(dag.findLCA(root, 1, 1), 1)
#Test 3: test when a value is not in tree
self.assertEqual(dag.findLCA(root, 50, 51), None)
#Test 4: Test dag with 7 nodes
root.children.append(dag.Node(2))
root.children.append(dag.Node(3))
root.children[0].children.append(dag.Node(4))
root.children[0].children.append(dag.Node(5))
root.children[1].children.append(dag.Node(6))
root.children[1].children[0].children.append(dag.Node(5))
root.children[1].children[0].children.append(dag.Node(7))
self.assertEqual(dag.findLCA(root, 4, 5), 2)
#Test 5: dag 4-6
self.assertEqual(dag.findLCA(root, 4, 6), 1)
#Test 6: dag 3-4
self.assertEqual(dag.findLCA(root, 3, 4), 1)
#Test 7: dag 2-4
self.assertEqual(dag.findLCA(root, 2, 4), 2)
#Test 8: nodes at different heights
self.assertAlmostEqual(dag.findLCA(root, 2, 6), 1)
#self.assertAlmostEqual(dag.findLCA(root, 6, 8), 3)
self.assertEqual(dag.findLCA(root, 6, 7), 6)
def test_find_lca_dag(self):
root = dag.Node(1)
r2 = dag.Node(2)
r3 = dag.Node(3)
r4 = dag.Node(4)
r5 = dag.Node(5)
r6 = dag.Node(6)
root.succ = [r2, r3]
r2.succ = [r5, r6]
r2.pred = [root]
r3.succ = [r4, r5]
r3.pred = [root]
r4.succ = [r5]
r4.pred = [r3]
r5.succ = [r6]
r5.pred = [r2, r3, r4]
r6.pred = [r2, r5]
#Testing lca of r4 and r5
lca_4_5 = dag.find_lca_dag(root, r4, r5)
assert lca_4_5 is 3
#Testing lca of r5 and r6
lca_5_6 = dag.find_lca_dag(root, r5, r6)
assert lca_5_6 is 2
#Testing lca of r4 and r6 - no immediate LCA
lca_4_6 = dag.find_lca_dag(root, r4, r6)
assert lca_4_6 is 1
#Testing lca of r2 and r3
lca_2_3 = dag.find_lca_dag(root, r2, r3)
assert lca_2_3 is 1
#Testing lca of root and root
lca_1_1 = dag.find_lca_dag(root, root, root)
assert lca_1_1 is 1
def _BuildDag(self, requests):
"""Build DAG of resources.
Build a DAG from our requests and augment with _NodeInfo (see above) in a
parallel array indexed by Node.Index().
Creates self._nodes and self._node_info.
Args:
requests: [Request, ...] Requests from loading.log_parser.
"""
self._nodes = []
self._node_info = []
indicies_by_url = {}
requests_by_completion = log_parser.SortedByCompletion(requests)
for request in requests:
next_index = len(self._nodes)
indicies_by_url.setdefault(request.url, []).append(next_index)
node = dag.Node(next_index)
node_info = self._NodeInfo(node, request)
self._nodes.append(node)
self._node_info.append(node_info)
for url, indicies in indicies_by_url.iteritems():
if len(indicies) > 1:
logging.warning('Multiple loads (%d) for url: %s' %
(len(indicies), url))
for i in xrange(len(requests)):
request = requests[i]
current_node_info = self._node_info[i]
resource = log_parser.Resource.FromRequest(current_node_info.Request())
initiator = request.initiator
initiator_type = initiator['type']
predecessor_url = None
predecessor_type = None
# Classify & infer the predecessor. If a candidate url we identify as the
# predecessor is not in index_by_url, then we haven't seen it in our
# requests and we will try to find a better predecessor.
if initiator_type == 'parser':
url = initiator['url']
if url in indicies_by_url:
predecessor_url = url
predecessor_type = 'parser'
elif initiator_type == 'script' and 'stackTrace' in initiator:
for frame in initiator['stackTrace']:
url = frame['url']
if url in indicies_by_url:
predecessor_url = url
predecessor_type = 'stack'
break
elif initiator_type == 'script':
# When the initiator is a script without a stackTrace, infer that it
# comes from the most recent script from the same hostname. TLD+1 might
# be better, but finding what is a TLD requires a database.
request_hostname = urlparse.urlparse(request.url).hostname
sorted_script_requests_from_hostname = [
r for r in requests_by_completion
if (resource.GetContentType() in ('script', 'html', 'json')
and urlparse.urlparse(r.url).hostname == request_hostname)]
most_recent = None
# Linear search is bad, but this shouldn't matter here.
for r in sorted_script_requests_from_hostname:
if r.timestamp < request.timing.requestTime:
most_recent = r
else:
break
if most_recent is not None:
url = most_recent.url
if url in indicies_by_url:
predecessor_url = url
predecessor_type = 'script_inferred'
# TODO(mattcary): we skip initiator type other, is that correct?
if predecessor_url is not None:
predecessor = self._FindBestPredecessor(
current_node_info, indicies_by_url[predecessor_url])
edge_cost = current_node_info.StartTime() - predecessor.EndTime()
if edge_cost < 0:
edge_cost = 0
if current_node_info.StartTime() < predecessor.StartTime():
logging.error('Inverted dependency: %s->%s',
predecessor.ShortName(), current_node_info.ShortName())
# Note that current.StartTime() < predecessor.EndTime() appears to
# happen a fair amount in practice.
predecessor.Node().AddSuccessor(current_node_info.Node())
predecessor.SetEdgeCost(current_node_info, edge_cost)
predecessor.AddEdgeAnnotation(current_node_info, predecessor_type)
