def set_process_ids(self):
vals = [process_id.label
for process_id in self.ordered_ids] # Store previous ids
for task in self.totals:
for pid in self.totals[task]:
for tid in self.totals[task][pid]:
label = task + "-pid:" + pid + "-tid:" + tid
job = get_job(task)
if job not in self.all_jobs:
self.all_jobs.append(job)
if label not in vals:
process_name = self.tasks[task].process_name
event_name = self.tasks[task].event
raw_event = self.tasks[task].raw_event
event_type = self.tasks[task].event_type
vals.append(label)
self.ordered_ids.append(
StackDataID(
job,
label,
task,
pid,
tid,
process_name,
event_name,
raw_event,
event_type,
))
self.ordered_ids = natural_sort(
self.ordered_ids, key=lambda process_id: process_id.label)
self.all_jobs = natural_sort(self.all_jobs)
def get_results_info(path, results_files):
"""Read processes and events from results file. Return a map bewteen processes and job names,
and list of all raw perf events."""
job_process_map = OrderedDict()
processes = {}
jobs = []
raw_events = []
for result_file in results_files:
full_filename = os.path.join(path, result_file)
job = result_file.rpartition(".results")[0]
jobs.append(job)
processes[job] = []
with open(full_filename) as infile:
for line in infile:
if is_results_file(line):
match = re.match("(.*proc(all|[0-9]+))_(.*)", line.strip())
job_process = match.group(1)
event = match.group(3)
process = re.sub(job + "_", "", job_process)
job = job.partition("_proc")[0]
if process not in processes[job]:
processes[job].append(process)
if event not in raw_events:
raw_events.append(event)
processes[job] = natural_sort(processes[job])
jobs = natural_sort(jobs)
for job in jobs:
job_process_map[job] = processes[job]
raw_events = natural_sort(raw_events)
return job_process_map, raw_events
def get_jobs(results_files):
# Read jobs from results file
jobs = []
for result_file in results_files:
job = result_file.partition(".results")[0]
jobs.append(job)
jobs = natural_sort(jobs)
return jobs
def get_processes(path, results_files): # Read processes from results files
processes = []
for result_file in results_files:
full_filename = os.path.join(path, result_file)
with open(full_filename) as infile:
for line in infile:
if is_results_file(line):
match = re.match("(.*proc(all|[0-9]+))_(.*)", line.strip())
name = match.group(1)
if name not in processes:
processes.append(name)
processes = natural_sort(processes)
return processes
def get_events(path, results_files): # Read events from results files
events = []
for result_file in results_files:
full_filename = os.path.join(path, result_file)
with open(full_filename) as infile:
for line in infile:
if is_results_file(line):
match = re.match("(.*proc(all|[0-9]+))_(.*)", line.strip())
event = match.group(3)
if event not in events:
events.append(event)
events = natural_sort(events)
return events
def modify_system_wide_process_ids(orig_file):
# Replace process and thread ids with sets of sequential ids, starting from zero
# split data files for each host into separate files for each physical core
threads = []
host = re.findall("host(\d+)_", orig_file)[0][0]
process_id_regex = re.compile("((all|[0-9]+)/(all|[0-9]+))")
with open(orig_file, "r") as result:
for line in result:
match = re.search(process_id_regex, line)
if match:
tid = match.group(3)
if tid not in threads:
threads.append(tid)
sorted_threads = natural_sort(threads)
tids = OrderedDict()
n = 0
for t in sorted_threads:
if t == "all":
tids[t] = t
else:
tids[t] = str(n)
n += 1
fs = {}
last_time = ""
start_time = ""
with open(orig_file, "r") as result:
for line in result:
match = re.search(process_id_regex, line)
if match:
pid = match.group(2)
tid = match.group(3)
line = re.sub("/" + tid, "/" + tids[tid], line)
new_file = re.sub("host" + host, "host" + host + "_proc" + pid,
orig_file)
if new_file not in fs:
fs[new_file] = open(new_file, "wb")
fs[new_file].write(start_time.encode())
fs[new_file].write(last_time.encode())
fs[new_file].write(line.encode())
elif line[0:2] == "t=":
last_time = line
for f in fs:
fs[f].write(line.encode())
elif re.match("start-time", line):
start_time = line
for f in fs:
fs[f].close()
remove(orig_file)
def modify_process_ids(orig_pid, orig_file):
# Replace process and thread ids with sets of sequential ids, starting from zero
threads = {}
process_id_regex = re.compile("((all|[0-9]+)/(all|[0-9]+))")
with open(orig_file, "r") as result:
for line in result:
match = re.search(process_id_regex, line)
if match:
tid = match.group(3)
if tid not in threads:
threads[tid] = tid
sorted_threads = OrderedDict(
natural_sort(threads.items(), key=lambda th: th[1]))
tids = OrderedDict()
n = 0
for t in sorted_threads:
if t == "all":
tids[t] = t
else:
tids[t] = str(n)
n += 1
fh, abs_path = mkstemp()
with open(abs_path, "wb") as new_file: # Output file is binary
with open(orig_file, "r") as result: # Input file is text
for line in result:
match = re.search(process_id_regex, line)
if match:
pid = match.group(2)
tid = match.group(3)
line = re.sub("/" + tid, "/" + tids[tid], line)
line = re.sub(pid + "/", orig_pid + "/", line)
new_file.write(line.encode())
else:
new_file.write(line.encode())
close(fh)
remove(orig_file)
move(abs_path, orig_file)
def make_svg(self):
imagewidth = self.image_settings.imagewidth
frameheight = self.image_settings.frameheight
fonttype = self.image_settings.fonttype
fontsize = self.image_settings.fontsize
xpad = self.image_settings.xpad
ypad1 = self.image_settings.ypad1
ypad2 = self.image_settings.ypad2
framepad = self.image_settings.framepad
imageheight = self.nlevels * (frameheight +
2 * framepad) + ypad1 + ypad2
widthpertime = float(imagewidth - 2 * xpad) / float(self.intervals)
self.min_time = sys.maxsize
self.max_time = -sys.maxsize
scale_time = float(imagewidth - 2 * xpad)
self.im.header(imagewidth, imageheight)
self.im.include(self.svg_scripts)
self.im.filled_rectangle(0,
0,
imagewidth,
imageheight,
"transparent",
extra='id="background_rect"')
black = self.im.allocate_color(0, 0, 0)
self.im.string_ttf(
black,
fonttype,
fontsize,
xpad,
imageheight - (ypad2 / 2),
" ",
"",
'id="details"',
)
y1 = ypad1
for pid in natural_sort(self.timelines.keys()):
for tid in natural_sort(self.timelines[pid].keys()):
y2 = y1 + frameheight
x1 = xpad
for func, start, end, count in self.timelines[pid][tid]:
self.min_time = min(self.min_time, float(start))
self.max_time = max(self.max_time, float(end))
escaped_func = re.sub("&", "&", func)
escaped_func = re.sub("<", "<", escaped_func)
escaped_func = re.sub(">", ">", escaped_func)
escaped_func = re.sub('"', """, escaped_func)
start = "{:.6f}".format(float(start))
end = "{:.6f}".format(float(end))
info = "pid:{}-tid:{} Max: {} (Start: {}, End: {})".format(
str(pid), str(tid), escaped_func, start, end)
nameattr = Attributes(info)
self.im.group_start(nameattr)
color = self.color_handler.color_map(
self.image_settings.colors, func)
x2 = x1 + count * widthpertime
self.im.filled_rectangle(
x1, y1, x2, y2, color,
'rx="0" ry="0" fill-opacity="0.9"')
self.im.group_end()
x1 = x2
y1 += frameheight + 2 * framepad
scale_time /= self.max_time - self.min_time
x1 = xpad
y1 = float(ypad1) / 2.0
count = 0
n_ticks = self.intervals / 10
for pid in sorted(self.timelines.keys()):
for tid in sorted(self.timelines[pid].keys()):
dt = (self.sample_rates[pid][tid][1][0] -
self.sample_rates[pid][tid][0][0])
t = self.sample_rates[pid][tid][0][0] - 0.5 * dt
value_range = (self.sample_rates[pid][tid][-1][0] -
self.sample_rates[pid][tid][0][0])
for time, rate in self.sample_rates[pid][tid]:
t1 = precision(t, value_range)
if count % n_ticks == 0:
y2 = y1 - (float(ypad1) / 8.0)
self.im.line(
x1,
y1,
x1,
y2,
extra='class="time_interval" title="' + t1 + '"',
)
if count == 0:
loc = "start"
else:
loc = "middle"
self.im.string_ttf(
black,
fonttype,
fontsize - 2,
x1,
y2 + 1,
t1,
loc=loc,
extra='font-style="italic"',
)
else:
y2 = y1 - (float(ypad1) / 16.0)
self.im.line(
x1,
y1,
x1,
y2,
extra='class="time_interval" title="' + t1 + '"',
)
x1 += widthpertime
t += dt
count += 1
break
break
t1 = precision(t, value_range)
y2 = y1 - (float(ypad1) / 8.0)
loc = "end"
self.im.string_ttf(black,
fonttype,
fontsize - 2,
x1,
y2 + 1,
t1,
loc=loc)
self.im.line(x1,
y1,
x1,
y2,
extra='class="time_interval" title="' + t1 + '"')
self.im.line(xpad, y1, x1, y1)
y1 = ypad1
for pid in sorted(self.timelines.keys()):
if pid in self.sample_rates:
for tid in sorted(self.timelines[pid].keys()):
if tid in self.sample_rates[pid]:
x1 = xpad
path_coords = []
for time, rate in self.sample_rates[pid][tid]:
norm_rate = 0.0
if self.max_sample_rate > 0.0:
norm_rate = rate / self.max_sample_rate
xi = x1 + 0.5 * widthpertime
yi = y1 + (1.0 - norm_rate) * frameheight
path_coords.append((xi, yi))
x1 += widthpertime
self.im.polyline(path_coords)
y1 += frameheight + 2 * framepad
else:
y1 += frameheight + 2 * framepad
y1 = ypad1
colors = {}
for pid in sorted(self.timelines.keys()):
if pid in self.secondary_events:
for tid in sorted(self.timelines[pid].keys()):
if tid in self.secondary_events[pid]:
n_events = len(self.secondary_events[pid][tid])
step = max(1, n_events // 400)
n = 0
x1 = xpad
for event, time in self.secondary_events[pid][tid]:
n = n + 1
if n % step != 0:
continue
if event in self.event_map:
event = self.event_map[event]
xi = x1 + (time - self.min_time) * scale_time
yi = y1
xj = xi + 3
yj = yi + frameheight
if event not in colors:
nc = len(colors)
colors[event] = cluster_plot_colours[nc]
color = colors[event]
t = "{:.6f}".format(float(time))
nameattr = Attributes(event + " " + t)
self.im.group_start(nameattr)
self.im.filled_rectangle(xi, yi, xj, yj, color,
'rx="0" ry="0"')
self.im.group_end()
y1 += frameheight + 2 * framepad
else:
y1 += frameheight + 2 * framepad
self.write_timelines()
def get_active_events(self):
return natural_sort([
event_definition.event for event_definition in self.active_events
])
def get_available_events(self):
return natural_sort([
event_definition.event
for event_definition in self.available_events
])