def nflxprofile_generate_flame_graph(file_path, range_start, range_end, package_name=False, flavor='standard'): try: f = get_file(file_path) profile = nflxprofile_pb2.Profile() profile.ParseFromString(f.read()) except TypeError: abort(500, 'Failed to parse profile.') finally: f.close() stack_processor = FLAVORS.get(flavor, StackProcessor) start_time = profile.start_time if range_start is not None: range_start = (math.floor(start_time) + range_start) if range_end is not None: range_end = (math.floor(start_time) + range_end) return get_flame_graph(profile, {}, range_start=range_start, range_end=range_end, package_name=package_name, stack_processor=stack_processor)
def cpuprofile_generate_flame_graph(file_path, range_start, range_end): f = get_file(file_path) chrome_profile = json.load(f) f.close() cpuprofiles = get_cpuprofiles(chrome_profile) # a chrome profile can contain multiple cpu profiles # using only the first one for now # TODO: add support for multiple cpu profiles profile = cpuprofiles[0] root_id = profile['nodes'][0]['id'] nodes = parse_nodes(profile) ignore_ids = get_meta_ids(nodes) start_time = profile['startTime'] if range_start is not None: adjusted_range_start = (math.floor(start_time / 1000000) + range_start) * 1000000 if range_end is not None: adjusted_range_end = (math.floor(start_time / 1000000) + range_end) * 1000000 return generate_flame_graph(nodes, root_id, profile['samples'], profile['timeDeltas'], profile['startTime'], adjusted_range_start, adjusted_range_end, ignore_ids)
def cpuprofile_read_offsets(file_path): f = get_file(file_path) chrome_profile = json.load(f) f.close() cpuprofiles = get_cpuprofiles(chrome_profile) # a chrome profile can contain multiple cpu profiles # using only the first one for now # TODO: add support for multiple cpu profiles profile = cpuprofiles[0] time_deltas = profile['timeDeltas'] samples = profile['samples'] idle_id = get_idle_id(profile['nodes']) start_time = profile['startTime'] end_time = profile['endTime'] offsets = [] current_time = start_time for index, delta in enumerate(time_deltas): current_time += delta if samples[index] != idle_id: offsets.append(current_time / 1000000) res = collections.namedtuple('offsets', ['start', 'end', 'offsets'])(start_time / 1000000, end_time / 1000000, offsets) return res
def nflxprofile_generate_differential_flame_graph(file_path, range_start, range_end): try: f = get_file(file_path) profile = nflxprofile_pb2.Profile() profile.ParseFromString(f.read()) except TypeError: abort(500, 'Failed to parse profile.') finally: f.close() start_time = profile.start_time if range_start is not None: range_start = (math.floor(start_time) + range_start) if range_end is not None: range_end = (math.floor(start_time) + range_end) return generate_flame_graph([profile], [0], [None], range_start, range_end)
def nflxprofile_generate_flame_graph(file_path, range_start, range_end): try: f = get_file(file_path) profile = nflxprofile_pb2.Profile() profile.ParseFromString(f.read()) except TypeError: abort(500, 'Failed to parse profile.') finally: f.close() start_time = profile.start_time if range_start is not None: adjusted_range_start = (math.floor(start_time) + range_start) if range_end is not None: adjusted_range_end = (math.floor(start_time) + range_end) return generate_flame_graph(profile.nodes, 0, profile.samples, profile.time_deltas, start_time, adjusted_range_start, adjusted_range_end, None)
def trace_event_read_offsets(file_path, mtime): try: f = get_file(file_path) profile = json.load(f) except JSONDecodeError: abort(500, 'Failed to parse profile.') finally: f.close() root_slices = [] events = {} offsets = [] (start_time, end_time) = get_time_range(file_path, mtime, profile) # process all events in the profile to extract root events for row in profile: key = str(row['pid']) + '_' + str(row['tid']) if row['ph'] == 'B' or row['ph'] == 'E': if row['ph'] == 'B': if key not in events: events[key] = {'ts': row['ts'], 'tts': row['tts'], 'children_count': 0} else: events[key]['children_count'] = events[key]['children_count'] + 1 elif row['ph'] == 'E': if events[key]['children_count'] > 0: events[key]['children_count'] = events[key]['children_count'] - 1 else: root_slices.append({'start': events[key]['ts'], 'cpu_start': events[key]['tts'], 'end': row['ts'], 'cpu_end': row['tts']}) del events[key] elif row['ph'] == 'X': if 'dur' in row and row['dur'] > 0 and 'tdur' in row and row['tdur'] > 0: if key not in events: # it's a root event root_slices.append({'start': row['ts'], 'cpu_start': row['tts'], 'end': row['ts'] + row['dur'], 'cpu_end': row['tts'] + row['tdur']}) # process each root event and generate time offsets based on frequency for s in root_slices: first_index = math.floor(s['start'] / u_sec_interval) * u_sec_interval last_index = math.ceil(s['end'] / u_sec_interval) * u_sec_interval # TODO: user cpu usage # usage = (s['cpu_end'] - s['cpu_start']) / (s['end'] - s['start']) for i in range(first_index, last_index, u_sec_interval): offsets.append(i / 1000000) res = collections.namedtuple('offsets', ['start', 'end', 'offsets'])(start_time / 1000000, end_time / 1000000, offsets) return res
def nflxprofile_generate_flame_graph(filename, range_start, range_end, profile=None): if not profile: file_path = join(config.PROFILE_DIR, filename) (f, mime) = get_file(file_path) profile = json.load(f) f.close() start_time = profile['startTime'] if range_start is not None: adjusted_range_start = (math.floor(start_time) + range_start) if range_end is not None: adjusted_range_end = (math.floor(start_time) + range_end) return generate_flame_graph(profile['nodes'], profile['samples'], profile['timeDeltas'], profile['startTime'], adjusted_range_start, adjusted_range_end, None)
def perf_read_offsets(file_path): start = float("+inf") end = float("-inf") offsets = [] f = get_file(file_path) stack = "" ts = -1 # process perf script output and search for two things: # - event_regexp: to identify event timestamps # - idle_regexp: for filtering idle stacks # this populates start, end, and offsets for line in f: if (line[0] == '#'): continue r = event_regexp.search(line) if (r): if (stack != ""): # process prior stack if (not idle_regexp.search(stack)): offsets.append(ts) # don't try to cache stacks (could be many Gbytes): stack = "" ts = float(r.group(1)) if (ts < start): start = ts stack = line.rstrip() else: stack += line.rstrip() # last stack if (not idle_regexp.search(stack)): offsets.append(ts) if (ts > end): end = ts f.close() res = collections.namedtuple('offsets', ['start', 'end', 'offsets'])(start, end, offsets) return res
def nflxprofile_readoffsets(file_path): try: f = get_file(file_path) profile = nflxprofile_pb2.Profile() profile.ParseFromString(f.read()) except TypeError: abort(500, 'Failed to parse profile.') finally: f.close() offsets = [] current_time = profile.start_time for delta in profile.time_deltas: current_time += delta offsets.append(current_time) res = collections.namedtuple('offsets', ['start', 'end', 'offsets'])(profile.start_time, profile.end_time, offsets) return res
def cpuprofile_generate_flame_graph(filename, range_start, range_end, profile=None): if not profile: file_path = join(config.PROFILE_DIR, filename) f = get_file(file_path) profile = json.load(f) f.close() nodes = parse_nodes(profile) ignore_ids = get_meta_ids(nodes) root = Node('root') root_id = profile['nodes'][0]['id'] # no range defined, just return the callgraph if range_start is None and range_end is None: generate_callgraph(root, root_id, nodes, []) return root samples = profile['samples'] time_deltas = profile['timeDeltas'] start_time = profile['startTime'] #end_time = profile['endTime'] adjusted_start = (math.floor(start_time / 1000000) + range_start) * 1000000 adjusted_end = (math.floor(start_time / 1000000) + range_end) * 1000000 current_time = start_time + time_deltas[0] stacks = {} generate_stacks(root_id, nodes, stacks, []) for index, sample in enumerate(samples): if index == (len(samples) - 1): # last sample break delta = time_deltas[index + 1] if delta < 0: # TODO: find a better way to deal with negative time deltas delta = 0 continue current_time += delta if sample not in ignore_ids: if current_time >= adjusted_start and current_time < adjusted_end: stack = stacks[sample] root.add(stack, delta) return root
def calculate_profile_range(filename): start = float("+inf") end = float("-inf") index_factor = 100 # save one timestamp per this many lines file_path = join(config.PROFILE_DIR, filename) # check for cached times mtime = getmtime(file_path) if file_path in stack_times: if mtime == stack_mtimes[file_path]: return stack_times[file_path] f = get_file(file_path) linenum = -1 stack_index[file_path] = [] for line in f: linenum += 1 # 1. Skip '#' comments # 2. Since we're only interested in the event summary lines, skip the # stack trace lines based on those that start with '\t'. This is a # performance optimization that avoids using the regexp needlessly, # and makes a large difference. if (line[0] == '#' or line[0] == '\t'): continue r = event_regexp.search(line) if (r): ts = float(r.group(1)) if ((linenum % index_factor) == 0): stack_index[file_path].append([linenum, ts]) if (ts < start): start = ts elif (ts > end): end = ts f.close() times = collections.namedtuple('range', ['start', 'end'])(floor(start), ceil(end)) stack_times[file_path] = times stack_mtimes[file_path] = mtime return times
def cpuprofile_read_offsets(file_path): f = get_file(file_path) chrome_profile = json.load(f) f.close() cpuprofiles = get_cpuprofiles(chrome_profile) offsets = [] start_time = None end_time = None for profile in cpuprofiles: time_deltas = profile['timeDeltas'] samples = profile['samples'] idle_id = get_idle_id(profile['nodes']) if start_time is None or profile['startTime'] < start_time: start_time = profile['startTime'] current_time = profile['startTime'] for index, delta in enumerate(time_deltas): current_time += delta if samples[index] != idle_id: offsets.append(current_time / 1000000) if 'endTime' in profile: if end_time is None or profile['endTime'] > end_time: end_time = profile['endTime'] else: if end_time is None or current_time > end_time: end_time = current_time res = collections.namedtuple('offsets', ['start', 'end', 'offsets'])( start_time / 1000000, end_time / 1000000, offsets) return res
def cpuprofile_generate_flame_graph(file_path, range_start, range_end): f = get_file(file_path) chrome_profile = json.load(f) f.close() profiles = get_cpuprofiles(chrome_profile) root_ids = [] ignore_ids = [] start_time = None for profile in profiles: root_ids.append(profile['nodes'][0]['id']) parsed_nodes = parse_nodes(profile['nodes']) ignore_ids.append(get_meta_ids(parsed_nodes)) profile['nodes'] = parsed_nodes if start_time is None or profile['startTime'] < start_time: start_time = profile['startTime'] if range_start is not None: adjusted_range_start = (math.floor(start_time / 1000000) + range_start) * 1000000 if range_end is not None: adjusted_range_end = (math.floor(start_time / 1000000) + range_end) * 1000000 return generate_flame_graph(profiles, root_ids, ignore_ids, adjusted_range_start, adjusted_range_end)
def cpuprofile_generate_flame_graph(filename, range_start, range_end, profile=None): if not profile: file_path = join(config.PROFILE_DIR, filename) (f, mime) = get_file(file_path) profile = json.load(f) f.close() nodes = parse_nodes(profile) ignore_ids = get_meta_ids(nodes) start_time = profile['startTime'] if range_start is not None: adjusted_range_start = (math.floor(start_time / 1000000) + range_start) * 1000000 if range_end is not None: adjusted_range_end = (math.floor(start_time / 1000000) + range_end) * 1000000 return generate_flame_graph(nodes, profile['samples'], profile['timeDeltas'], profile['startTime'], adjusted_range_start, adjusted_range_end, ignore_ids)
def trace_event_generate_flame_graph(file_path, mtime, range_start, range_end): try: f = get_file(file_path) profile = json.load(f) except JSONDecodeError: abort(500, 'Failed to parse profile.') finally: f.close() root = {'name': 'root', 'value': 0, 'children': []} open_partial_slices = {} (start_time, end_time) = get_time_range(file_path, mtime, profile) adjusted_start = (math.floor(start_time / 1000000) + range_start) * 1000000 adjusted_end = (math.floor(start_time / 1000000) + range_end) * 1000000 def filter_and_adjust_slice_times(profile_slice): # slice starts after the range # slice ends before the range if profile_slice['ts'] > adjusted_end or ( profile_slice['ts'] + profile_slice['dur']) < adjusted_start: return None # slice starts before the range, need to adjust start time and duration if profile_slice['ts'] < adjusted_start: ts_diff = profile_slice['ts'] - adjusted_start profile_slice['ts']: adjusted_start profile_slice['dur'] = profile_slice['dur'] + ts_diff # slice ends after the range, need to adjust duration if (profile_slice['ts'] + profile_slice['dur']) > adjusted_end: ts_diff = adjusted_end - (profile_slice['ts'] + profile_slice['dur']) profile_slice['dur'] = profile_slice['dur'] + ts_diff # filter children if len(profile_slice['children']) > 0: filtered_children = [] for child in profile_slice['children']: filtered_child = filter_and_adjust_slice_times(child) if filtered_child is not None: filtered_children.append(filtered_child) profile_slice['children'] = filtered_children return profile_slice def get_child_slice(parent_slice, name): for index, child in enumerate(parent_slice['children']): if child['name'] == name: return parent_slice['children'].pop(index) return None def insert_slice(parent_slice, new_slice): child_slice = get_child_slice(parent_slice, new_slice['name']) if child_slice is None: child_slice = { 'name': new_slice['name'], 'value': 0, 'children': [] } for child in new_slice['children']: insert_slice(child_slice, child) child_slice['value'] += new_slice['value'] parent_slice['children'].append(child_slice) def check_thread(pid, tid): if pid not in open_partial_slices: open_partial_slices[pid] = {} if tid not in open_partial_slices[pid]: open_partial_slices[pid][tid] = [] def begin_slice(pid, tid, cat, name, ts, tts): check_thread(pid, tid) open_partial_slices[pid][tid].append({ 'pid': pid, 'tid': tid, 'cat': cat, 'name': name, 'ts': ts, 'tts': tts, 'children': [] }) def end_slice(pid, tid, ts, tts): partial_slice_count = len(open_partial_slices[pid][tid]) if partial_slice_count > 0: current_slice = open_partial_slices[pid][tid].pop() current_slice['dur'] = ts - current_slice['ts'] current_slice['tdur'] = tts - current_slice['tts'] if current_slice['dur'] > 0: current_slice[ 'value'] = current_slice['tdur'] / current_slice['dur'] partial_slice_count = len(open_partial_slices[pid][tid]) if partial_slice_count > 0: open_partial_slices[pid][tid][ partial_slice_count - 1]['children'].append(current_slice) else: filtered_slice = filter_and_adjust_slice_times(current_slice) if filtered_slice is not None: insert_slice(root, current_slice) else: raise Exception("end_slice called without an open slice") for row in profile: if row['ph'] == 'B' or row['ph'] == 'E': if row['ph'] == 'B': begin_slice(row['pid'], row['tid'], row['cat'], row['name'], row['ts'], row['tts']) elif row['ph'] == 'E': end_slice(row['pid'], row['tid'], row['ts'], row['tts']) elif row['ph'] == 'X': if 'dur' in row and row['dur'] > 0 and 'tdur' in row and row[ 'tdur'] > 0: begin_slice(row['pid'], row['tid'], row['cat'], row['name'], row['ts'], row['tts']) end_slice(row['pid'], row['tid'], row['ts'] + row['dur'], row['tts'] + row['tdur']) return root
def perf_generate_flame_graph(filename, range_start=None, range_end=None): file_path = join(config.PROFILE_DIR, filename) (f, mime) = get_file(file_path) # calculate profile file range r = calculate_profile_range(filename) start = r.start end = r.end # check range. default to full range if not specified. if (range_end): if ((start + range_end) > end): print("ERROR: Bad range, %s -> %s." % (str(start), str(end))) return abort(416) else: end = start + range_end if (range_start): start = start + range_start if (start > end): print("ERROR: Bad range, %s -> %s." % (str(start), str(end))) return abort(416) root = {} root['c'] = [] root['n'] = "root" root['l'] = "" root['v'] = 0 stack = [] ts = -1 comm = "" # overscan is seconds beyond the time range to keep scanning, which allows # for some out-of-order samples up to this duration overscan = 0.1 # determine skip lines lastline = 0 skiplines = 0 if file_path in stack_index: for pair in stack_index[file_path]: if start < pair[1]: # scanned too far, use last entry skiplines = lastline break lastline = pair[0] # process perf script output and search for two things: # - event_regexp: to identify event timestamps # - idle_regexp: for filtering idle stacks linenum = -1 for line in f: linenum += 1 # Performance optimization. Makes a large difference. if (linenum < skiplines): continue # skip comments if (line[0] == '#'): continue # As a performance optimization, skip an event regexp search if the # line looks like a stack trace based on starting with '\t'. This # makes a big difference. r = None if (line[0] != '\t'): r = event_regexp.search(line) if (r): if (stack): # process prior stack stackstr = "" for pair in stack: stackstr += pair[0] + ";" if (idle_regexp.search(stackstr)): # skip idle stack = [] elif (ts >= start and ts <= end): root = add_stack(root, stack, comm) stack = [] ts = float(r.group(1)) if (ts > end + overscan): break r = comm_regexp.search(line) if (r): comm = r.group(1).rstrip() stack.append([comm, ""]) else: stack.append(["<unknown>", ""]) else: r = frame_regexp.search(line) if (r): name = r.group(1) # strip instruction offset (+0xfe200...) c = name.find("+") if (c > 0): name = name[:c] stack.insert(1, [name, r.group(2)]) # last stack if (ts >= start and ts <= end): root = add_stack(root, stack, comm) # close file f.close() return root