def _AugmentFrameLoads(self, index_by_request): if not self._frame_lens: return loads = self._frame_lens.GetFrameLoadInfo() load_index_to_node = {} for l in loads: next_index = len(self._nodes) node = dag.Node(next_index) node_info = self._NodeInfo(node, (l.index, l.msec)) load_index_to_node[l.index] = next_index self._nodes.append(node) self._node_info.append(node_info) frame_deps = self._frame_lens.GetFrameLoadDependencies() for load_idx, rq in frame_deps[0]: parent = self._node_info[load_index_to_node[load_idx]] child = self._node_info[index_by_request[rq]] parent.Node().AddSuccessor(child.Node()) parent.AddEdgeAnnotations(child, {self.EDGE_KIND_KEY: 'after-load'}) for rq, load_idx in frame_deps[1]: child = self._node_info[load_index_to_node[load_idx]] parent = self._node_info[index_by_request[rq]] parent.Node().AddSuccessor(child.Node()) parent.AddEdgeAnnotations(child, {self.EDGE_KIND_KEY: 'before-load'})
def testfindLCA(self): root = Node(1) root.children.append(dag.Node(2)) root.children.append(dag.Node(3)) root.children[0].children.append(dag.Node(4)) root.children[0].children.append(dag.Node(5)) root.children[1].children.append(dag.Node(6)) root.children[1].children.append(dag.Node(7)) # test all nodes are None self.assertEqual(dag.findLCA(None, None, None), None) #test when root is euqal to node self.assertEqual(dag.findLCA(root, 1, 1), 1) #test if n1 is not present self.assertEqual(dag.findLCA(root, None, 2), None) #test if n2 is not present self.assertEqual(dag.findLCA(root, 2, None), None) #test when node is value is not in tree self.assertEqual(dag.findLCA(root, 2, 13), None) #test dag for different values self.assertEqual(dag.findLCA(root, 3, 7), 3) self.assertEqual(dag.findLCA(root, 4, 6), 1) self.assertEqual(dag.findLCA(root, 2, 5), 2) self.assertEqual(dag.findLCA(root, 3, 4), 1)
def test_find_lca_dag_input_error(self): root = dag.Node(1) r2 = dag.Node(2) r3 = dag.Node(3) r4 = dag.Node(4) r5 = dag.Node(5) r6 = dag.Node(6) root.succ = [r2, r3] r2.succ = [r5, r6] r2.pred = [root] r3.succ = [r4, r5] r3.pred = [root] r4.succ = [r5] r4.pred = [r3] r5.succ = [r6] r5.pred = [r2, r3, r4] r6.pred = [r2, r5] lca_invalid_root = dag.find_lca_dag("a", r2, r3) assert lca_invalid_root is None lca_invalid_n1 = dag.find_lca_dag(root, "a", r3) assert lca_invalid_n1 is None lca_invalid_n2 = dag.find_lca_dag(root, r2, "a") assert lca_invalid_n2 is None
def _BuildDag(self, trace): """Build DAG of resources. Build a DAG from our requests and augment with NodeInfo (see above) in a parallel array indexed by Node.Index(). Creates self._nodes and self._node_info. Args: trace: A LoadingTrace. """ self._nodes = [] self._node_info = [] index_by_request = {} for request in trace.request_track.GetEvents(): next_index = len(self._nodes) assert request not in index_by_request index_by_request[request] = next_index node = dag.Node(next_index) node_info = self._NodeInfo(node, request) if self._content_lens: node_info.SetRequestContent( self._content_lens.IsAdRequest(request), self._content_lens.IsTrackingRequest(request)) self._nodes.append(node) self._node_info.append(node_info) dependencies = self.REQUEST_LENS(trace).GetRequestDependencies() for dep in dependencies: (parent_rq, child_rq, reason) = dep parent = self._node_info[index_by_request[parent_rq]] child = self._node_info[index_by_request[child_rq]] edge_cost = request_track.TimeBetween(parent_rq, child_rq, reason) if edge_cost < 0: edge_cost = 0 if child.StartTime() < parent.StartTime(): logging.error('Inverted dependency: %s->%s', parent.ShortName(), child.ShortName()) # Note that child.StartTime() < parent.EndTime() appears to happen a # fair amount in practice. parent.Node().AddSuccessor(child.Node()) parent.SetEdgeCost(child, edge_cost) parent.AddEdgeAnnotations(child, {self.EDGE_KIND_KEY: reason}) if self._activity_lens: activity = self._activity_lens.BreakdownEdgeActivityByInitiator( dep) parent.AddEdgeAnnotations(child, {'activity': activity}) self._AugmentFrameLoads(index_by_request)
def test_pathTo(self): #Graph with 7 nodes root = dag.Node(1) root.children.append(dag.Node(2)) root.children.append(dag.Node(3)) root.children[0].children.append(dag.Node(4)) root.children[0].children.append(dag.Node(5)) root.children[1].children.append(dag.Node(6)) root.children[1].children[0].children.append(dag.Node(5)) root.children[1].children[0].children.append(dag.Node(7)) #Test 1: Test when root is None self.assertEqual(dag.pathTo(None, 2, 3), False) #Test 2: Test all values are None self.assertEqual(dag.pathTo(None, None, None), False) #Test 3: test path to root self.assertEqual(dag.pathTo(root, [], 1), True) #Test 4: test for key not in tree self.assertEqual(dag.pathTo(root, [], 25), False) #Test 4: path to 7 self.assertEqual(dag.pathTo(root, [], 7), True) #Test 6: path to 6 self.assertEqual(dag.pathTo(root, [], 6), True) #Test 7: path to 5 self.assertEqual(dag.pathTo(root, [], 5), True) #Test 8: path to 4 self.assertEqual(dag.pathTo(root, [], 4), True) #Test 9: path to 3 self.assertEqual(dag.pathTo(root, [], 3), True) #Test 10: path to 2 self.assertEqual(dag.pathTo(root, [], 2), True)
def MakeDag(self, links): """Make a graph from a description of links. Args: links: A list of (index, (successor index...)) tuples. Index must equal the location of the tuple in the list and are provided to make it easier to read. Returns: A list of Nodes. """ nodes = [] for i in xrange(len(links)): assert i == links[i][0] nodes.append(dag.Node(i)) for l in links: for s in l[1]: nodes[l[0]].AddSuccessor(nodes[s]) return nodes
def test_findPathsTo(self): #new Graph root = dag.Node(1) root.children.append(dag.Node(2)) root.children.append(dag.Node(3)) root.children[0].children.append(dag.Node(4)) root.children[1].children.append(dag.Node(6)) root.children[1].children[0].children.append(dag.Node(5)) root.children[1].children[0].children.append(dag.Node(7)) #Test 1: test when root is equal None self.assertEqual(dag.findPathsTo(None, 2), []) #Test 2: test when key is None self.assertEqual(dag.findPathsTo(root, None), []) #Test 3: test when all keys are None self.assertEqual(dag.findPathsTo(None, None), []) #Test 4: test path to root self.assertEqual(dag.findPathsTo(root, 1), [[1]]) #Test 5: test for key not in tree self.assertEqual(dag.findPathsTo(root, 25), []) #Test 6: paths to 7 self.assertEqual(dag.findPathsTo(root, 7), [[1, 3, 6, 7]]) #Test 7: path to 5 self.assertEqual(dag.findPathsTo(root, 5), [[1, 3, 6, 5]]) #Test 8: path to 4 self.assertEqual(dag.findPathsTo(root, 4), [[1, 2, 4]]) #Test 9: test for more than one path root.children[0].children[0].children.append(dag.Node(5)) self.assertEqual(dag.findPathsTo(root, 5), [[1, 2, 4, 5], [1, 3, 6, 5]])
def test_findLCA_v2(self): #Test 1: test all values equal None self.assertEqual(dag.findLCA(None, None, None), None) #Test 2: test when both node are root root = dag.Node(1) self.assertEqual(dag.findLCA(root, 1, 1), 1) #Test 3: test when a value is not in tree self.assertEqual(dag.findLCA(root, 50, 51), None) #Test 4: Test dag with 7 nodes root.children.append(dag.Node(2)) root.children.append(dag.Node(3)) root.children[0].children.append(dag.Node(4)) root.children[0].children.append(dag.Node(5)) root.children[1].children.append(dag.Node(6)) root.children[1].children[0].children.append(dag.Node(5)) root.children[1].children[0].children.append(dag.Node(7)) self.assertEqual(dag.findLCA(root, 4, 5), 2) #Test 5: dag 4-6 self.assertEqual(dag.findLCA(root, 4, 6), 1) #Test 6: dag 3-4 self.assertEqual(dag.findLCA(root, 3, 4), 1) #Test 7: dag 2-4 self.assertEqual(dag.findLCA(root, 2, 4), 2) #Test 8: nodes at different heights self.assertAlmostEqual(dag.findLCA(root, 2, 6), 1) #self.assertAlmostEqual(dag.findLCA(root, 6, 8), 3) self.assertEqual(dag.findLCA(root, 6, 7), 6)
def test_find_lca_dag(self): root = dag.Node(1) r2 = dag.Node(2) r3 = dag.Node(3) r4 = dag.Node(4) r5 = dag.Node(5) r6 = dag.Node(6) root.succ = [r2, r3] r2.succ = [r5, r6] r2.pred = [root] r3.succ = [r4, r5] r3.pred = [root] r4.succ = [r5] r4.pred = [r3] r5.succ = [r6] r5.pred = [r2, r3, r4] r6.pred = [r2, r5] #Testing lca of r4 and r5 lca_4_5 = dag.find_lca_dag(root, r4, r5) assert lca_4_5 is 3 #Testing lca of r5 and r6 lca_5_6 = dag.find_lca_dag(root, r5, r6) assert lca_5_6 is 2 #Testing lca of r4 and r6 - no immediate LCA lca_4_6 = dag.find_lca_dag(root, r4, r6) assert lca_4_6 is 1 #Testing lca of r2 and r3 lca_2_3 = dag.find_lca_dag(root, r2, r3) assert lca_2_3 is 1 #Testing lca of root and root lca_1_1 = dag.find_lca_dag(root, root, root) assert lca_1_1 is 1
def _BuildDag(self, requests): """Build DAG of resources. Build a DAG from our requests and augment with _NodeInfo (see above) in a parallel array indexed by Node.Index(). Creates self._nodes and self._node_info. Args: requests: [Request, ...] Requests from loading.log_parser. """ self._nodes = [] self._node_info = [] indicies_by_url = {} requests_by_completion = log_parser.SortedByCompletion(requests) for request in requests: next_index = len(self._nodes) indicies_by_url.setdefault(request.url, []).append(next_index) node = dag.Node(next_index) node_info = self._NodeInfo(node, request) self._nodes.append(node) self._node_info.append(node_info) for url, indicies in indicies_by_url.iteritems(): if len(indicies) > 1: logging.warning('Multiple loads (%d) for url: %s' % (len(indicies), url)) for i in xrange(len(requests)): request = requests[i] current_node_info = self._node_info[i] resource = log_parser.Resource.FromRequest(current_node_info.Request()) initiator = request.initiator initiator_type = initiator['type'] predecessor_url = None predecessor_type = None # Classify & infer the predecessor. If a candidate url we identify as the # predecessor is not in index_by_url, then we haven't seen it in our # requests and we will try to find a better predecessor. if initiator_type == 'parser': url = initiator['url'] if url in indicies_by_url: predecessor_url = url predecessor_type = 'parser' elif initiator_type == 'script' and 'stackTrace' in initiator: for frame in initiator['stackTrace']: url = frame['url'] if url in indicies_by_url: predecessor_url = url predecessor_type = 'stack' break elif initiator_type == 'script': # When the initiator is a script without a stackTrace, infer that it # comes from the most recent script from the same hostname. TLD+1 might # be better, but finding what is a TLD requires a database. request_hostname = urlparse.urlparse(request.url).hostname sorted_script_requests_from_hostname = [ r for r in requests_by_completion if (resource.GetContentType() in ('script', 'html', 'json') and urlparse.urlparse(r.url).hostname == request_hostname)] most_recent = None # Linear search is bad, but this shouldn't matter here. for r in sorted_script_requests_from_hostname: if r.timestamp < request.timing.requestTime: most_recent = r else: break if most_recent is not None: url = most_recent.url if url in indicies_by_url: predecessor_url = url predecessor_type = 'script_inferred' # TODO(mattcary): we skip initiator type other, is that correct? if predecessor_url is not None: predecessor = self._FindBestPredecessor( current_node_info, indicies_by_url[predecessor_url]) edge_cost = current_node_info.StartTime() - predecessor.EndTime() if edge_cost < 0: edge_cost = 0 if current_node_info.StartTime() < predecessor.StartTime(): logging.error('Inverted dependency: %s->%s', predecessor.ShortName(), current_node_info.ShortName()) # Note that current.StartTime() < predecessor.EndTime() appears to # happen a fair amount in practice. predecessor.Node().AddSuccessor(current_node_info.Node()) predecessor.SetEdgeCost(current_node_info, edge_cost) predecessor.AddEdgeAnnotation(current_node_info, predecessor_type)