def ExtractParserDiscoverableResources(loading_trace_path): """Extracts the parser discoverable resources from a loading trace. Args: loading_trace_path: The loading trace's path. Returns: A set of urls. """ whitelisted_urls = set() logging.info('loading %s' % loading_trace_path) trace = loading_trace.LoadingTrace.FromJsonFile(loading_trace_path) requests_lens = request_dependencies_lens.RequestDependencyLens(trace) deps = requests_lens.GetRequestDependencies() main_resource_request = deps[0][0] logging.info('white-listing %s' % main_resource_request.url) whitelisted_urls.add(main_resource_request.url) for (first, second, reason) in deps: # Work-around where the protocol may be none for an unclear reason yet. # TODO(gabadie): Follow up on this with Clovis guys and possibly remove # this work-around. if not second.protocol: logging.info('ignoring %s (no protocol)' % second.url) continue # Ignore data protocols. if not second.protocol.startswith('http'): logging.info('ignoring %s (`%s` is not HTTP{,S} protocol)' % ( second.url, second.protocol)) continue if (first.request_id == main_resource_request.request_id and reason == 'parser' and second.url not in whitelisted_urls): logging.info('white-listing %s' % second.url) whitelisted_urls.add(second.url) return whitelisted_urls
def _AllCores(prefix, graph_set_names, output, threshold): """Compute all core sets (per-set and overall page core) for a site.""" core_sets = [] _Progress('Using threshold %s' % threshold) big_sack = resource_sack.GraphSack() graph_sets = [] for name in graph_set_names: _Progress('Finding core set for %s' % name) sack = resource_sack.GraphSack() sack.CORE_THRESHOLD = threshold this_set = [] for filename in glob.iglob('-'.join([prefix, name, '*.trace'])): _Progress('Reading %s' % filename) trace = loading_trace.LoadingTrace.FromJsonFile(filename) graph = dependency_graph.RequestDependencyGraph( trace.request_track.GetEvents(), request_dependencies_lens.RequestDependencyLens(trace)) sack.ConsumeGraph(graph) big_sack.ConsumeGraph(graph) this_set.append(graph) core_sets.append({ 'set_name': name, 'core_set': [l for l in sack.CoreSet()] }) graph_sets.append(this_set) json.dump( { 'core_sets': core_sets, 'page_core': [l for l in big_sack.CoreSet(*graph_sets)] }, output, sort_keys=True, indent=2)
def _SetUp(self, added_trace_events=None): trace_events = [ {'ts': 5, 'ph': 'X', 'dur': 10, 'pid': 2, 'tid': 1, 'cat': 'blink.net'}] if added_trace_events is not None: trace_events += added_trace_events self.trace = TestRequests.CreateLoadingTrace(trace_events) self.dependencies_lens = request_dependencies_lens.RequestDependencyLens( self.trace) self.user_satisfied_lens = test_utils.MockUserSatisfiedLens(self.trace) self.user_satisfied_lens._postload_msec = 12 self.prefetch_view = PrefetchSimulationView( self.trace, self.dependencies_lens, self.user_satisfied_lens) for e in self.prefetch_view.graph.graph.Edges(): e.cost = 10
def FromTrace(cls, trace): """Create a graph from a trace with no additional annotation.""" return cls(trace, request_dependencies_lens.RequestDependencyLens(trace))