timestamps = dict()
value = dict()
def isnum(x):
return re.match(r'[0-9.]+', x)
val = ""
for row in rc:
r = csv_formats.parse_csv_row(row)
if r is None:
continue
ts, cpu, event, val = r.ts, r.cpu, r.ev, r.val
if event not in assigned:
level = gen_level.get_level(event)
assigned[event] = cur_colors[level][0]
cur_colors[level] = cur_colors[level][1:]
if len(cur_colors[level]) == 0:
cur_colors[level] = all_colors
value[event] = []
timestamps[event] = []
timestamps[event].append(float(ts))
try:
value[event].append(float(val.replace("%", "")))
except ValueError:
value[event].append(0.0)
levels = set(map(gen_level.get_level, assigned.keys()))
if args.xkcd:
timestamps = dict()
value = dict()
def isnum(x):
return re.match(r'[0-9.]+', x)
val = ""
for row in rc:
r = csv_formats.parse_csv_row(row)
if r is None:
continue
ts, cpu, event, val = r.ts, r.cpu, r.ev, r.val
if event not in assigned:
level = gen_level.get_level(event)
assigned[event] = cur_colors[level][0]
cur_colors[level] = cur_colors[level][1:]
if len(cur_colors[level]) == 0:
cur_colors[level] = all_colors
value[event] = []
timestamps[event] = []
timestamps[event].append(float(ts))
try:
value[event].append(float(val.replace("%", "")))
except ValueError:
value[event].append(0.0)
levels = set(map(gen_level.get_level, assigned.keys()))
if args.xkcd:
val = ""
for r in rc:
# timestamp,event,value
if len(r) < 3:
continue
print r
if len(r) >= 5 and not isnum(r[1]):
ts, event, val, thresh, desc = r[:5]
elif len(r) >= 4:
ts, val, unit, event = r[:4]
else:
ts, val, event = r
if event not in assigned:
level = gen_level.get_level(event)
assigned[event] = cur_colors[level][0]
cur_colors[level] = cur_colors[level][1:]
if len(cur_colors[level]) == 0:
cur_colors[level] = all_colors
value[event] = []
timestamps[event] = []
timestamps[event].append(float(ts))
try:
value[event].append(float(val.replace("%", "")))
except ValueError:
value[event].append(0.0)
levels = dict()
for j in assigned.keys():
levels[gen_level.get_level(j)] = True
return re.match(r'[0-9.]+', x)
val = ""
for r in rc:
# timestamp,event,value
if len(r) < 3:
continue
print r
if len(r) >= 5 and not isnum(r[1]):
ts, event, val, thresh, desc = r[:5]
elif len(r) >= 4:
ts, val, unit, event = r[:4]
else:
ts, val, event = r
if event not in assigned:
level = gen_level.get_level(event)
assigned[event] = cur_colors[level][0]
cur_colors[level] = cur_colors[level][1:]
if len(cur_colors[level]) == 0:
cur_colors[level] = all_colors
value[event] = []
timestamps[event] = []
timestamps[event].append(float(ts))
try:
value[event].append(float(val.replace("%","")))
except ValueError:
value[event].append(0.0)
levels = dict()
for j in assigned.keys():
levels[gen_level.get_level(j)] = True
ratios = defaultdict(list)
timestamps = []
rc = csv.reader(open(args.file, "r"))
ts = None
levels = defaultdict(list)
vals = None
for r in rc:
if len(r) < 4:
continue
if not re.match(r"\d+(\.\d*)", r[0]):
r = ["0.0"] + r
if r[3] == "metric":
l = METRIC_LEVEL # put at end
else:
l = gen_level.get_level(r[1])
if r[1] not in levels[l]:
levels[l].append(r[1])
t = math.trunc(float(r[0]) * 100) / 100.0
if t != ts:
timestamps.append(t)
ts = t
flush_vals(ratios, vals)
vals = dict()
vals[r[1]] = float(r[2].replace("%",""))
flush_vals(ratios, vals)
if args.verbose:
print "time", len(timestamps), timestamps
for j in ratios.keys():
print j, ratios[j]
if len(sys.argv) > 1:
inf = open(sys.argv[1], "r")
else:
inf = sys.stdin
rc = csv.reader(inf)
timestamps = dict()
value = dict()
for r in rc:
# timestamp,event,value
if len(r) < 3:
continue
event = r[1]
if event not in assigned:
level = gen_level.get_level(event)
print level, event
assigned[event] = cur_colors[level][0]
cur_colors[level] = cur_colors[level][1:]
if len(cur_colors[level]) == 0:
cur_colors[level] = all_colors
value[event] = []
timestamps[event] = []
timestamps[event].append(r[0])
try:
value[event].append(float(r[2].replace("%","")))
except ValueError:
value[event].append(0.0)
levels = dict()
for j in assigned.keys():
rc = csv.reader(inf)
timestamps = dict()
value = dict()
def isnum(x):
return re.match(r'[0-9.]+', x)
val = ""
for row in rc:
r = csv_formats.parse_csv_row(row)
if r is None:
continue
ts, cpu, event, val = r.ts, r.cpu, r.ev, r.val
if event not in assigned:
level = gen_level.get_level(event)
assigned[event] = cur_colors[level][0]
cur_colors[level] = cur_colors[level][1:]
if len(cur_colors[level]) == 0:
cur_colors[level] = all_colors
value[event] = []
timestamps[event] = []
timestamps[event].append(float(ts))
try:
value[event].append(float(val.replace("%","")))
except ValueError:
value[event].append(0.0)
levels = set(map(gen_level.get_level, assigned.keys()))
if args.xkcd:
def plot(files, pstyle = 'ggplot', output=None, seq=None, xkcd=False):
global csv_file_handle
global value
csv_file_handle ={}
# op_sum = {'1':['L1-dcache-loads','L1-dcache-stores','L1-dcache-prefetches','L1-icache-loads'],
# '2':['L1-dcache-load-misses','L1-dcache-store-misses','L1-dcache-prefetch-misses','L1-icache-load-misses'],
# '3':[ 'LLC-loads','LLC-stores','LLC-prefetches'],
# '4':['LLC-load-misses','LLC-store-misses','LLC-prefetch-misses'],
# '5':['dTLB-loads','dTLB-stores','iTLB-loads'],
# '6':['dTLB-load-misses','dTLB-store-misses','iTLB-load-misses'],
# 'Bandwidth':['offcore_response_corewb_local_dram_0','offcore_response_prefetch_any_llc_miss_0','LLC-prefetches','cache-misses']}
#
# op_div = [['cache-references','uops_retired_any'],['cache-misses','uops_retired_any'], ['instructions','cycles'],
# ['cache-misses','cache-references']]
#enable for i7
op_sum = {
'contention': ['cache-misses'],
'band': ['cache-references', 'cache-misses'],
'total_bandwidth': ['cache-references'],
}
op_diff ={}
op_div= [['cache-references','uops_retired_any'],['cache-misses','uops_retired_any'],
['instructions','cycles'],['cache-misses','cache-references'],['cache-references','cycles'], ['cache-misses','cycles']]
print pstyle
if pstyle:
try:
from mpltools import style
style.use( pstyle)
except ImportError:
print "Need mpltools for setting styles (pip install mpltools)"
import gen_level
try:
import brewer2mpl
all_colors = brewer2mpl.get_map('Paired', 'Qualitative', 12).hex_colors
except ImportError:
print "Install brewer2mpl for better colors (pip install brewer2mpl)"
all_colors = ('green','orange','red','blue',
'black','olive','purple','#6960EC', '#F0FFFF',
'#728C00', '#827B60', '#F87217', '#E55451', # 16
'#F88017', '#C11B17', '#17BFC2', '#C48793') # 20
cur_colors = collections.defaultdict(lambda: all_colors)
assigned = dict() # assigned= {'mbw-cache-references': [0,2345,..], 'soplex-cache-references': [32,532,12,..], ..} Events and values for all processes
if len(files) < 2 :
print "More than one file needed. Exiting!"
sys.exit(0)
for file in files:
processname = file.split("/")[-1]
if file:
try:
inf = open( file, "r")
except:
return
else:
inf = sys.stdin
csv_file_handle[processname] = csv.reader(inf)
timestamps = dict()
val = ""
first_time = True
event_list = [] # event_list= [cache-references, instructions,..]
for processname,rc in csv_file_handle.items():
for r in rc:
if config.burst:
if len(r) == 2:
ts=0
val, event = r
if first_time and event not in event_list:
event_list.append(event)
event = str(processname)+"-"+event
else:
continue
if event not in assigned:
level = gen_level.get_level(event)
assigned[event] = cur_colors[level][0]
cur_colors[level] = cur_colors[level][1:]
if len(cur_colors[level]) == 0:
cur_colors[level] = all_colors
value[event] = []
timestamps[event] = []
timestamps[event].append(float(ts))
try:
value[event].append(float(val.replace("%","")))
except ValueError:
value[event].append(0.0)
first_time = False
levels = dict()
for j in assigned.keys():
levels[gen_level.get_level(j)] = True
if xkcd:
try:
plt.xkcd()
except NameError:
print "Please update matplotlib. Cannot enable xkcd mode."
#print value
if config.normalize:
for key in value:
entries= value[key]
normalized_values = [numpy.float64(entry)/max(entries) for entry in entries]
value[key] = normalized_values
if seq:
os.umask(0000)
if os.path.exists(seq):
shutil.rmtree(seq)
os.makedirs(seq)
else:
os.makedirs(seq)
n = 1
print "Assigned Keys: ", assigned.keys()
#print "event list: ", event_list
for l in levels.keys():
ax = plt.subplot(len(levels), 1, n)
if val.find('%') >= 0:
ax.set_ylim(0, 100)
t = []
for j in event_list:
print j, gen_level.get_level(j), l
for processname in csv_file_handle:
if gen_level.get_level(j) == l:
t.append(j)
ax.plot(value[str(processname)+"-"+j], label = str(processname)+"-"+j )
if seq:
leg = ax.legend( loc='upper left')
leg.get_frame().set_alpha(0.5)
plt.savefig(seq+"/"+j)
plt.cla()
leg = ax.legend(t, loc='upper left')
leg.get_frame().set_alpha(0.5)
n += 1
if len(op_diff) > 0:
for key, components in op_diff.items():
print components
#print [(value[component]) for component in components]
#print [len(value[component]) for component in components]
diff_value={}
if key =='contention' :
print "KEY: ", key
ax1 = plt.subplot(2,1,1)
ax2 = plt.subplot(2,1,2)
else:
ax = plt.subplot(1, 1, 1)
for processname in csv_file_handle:
diff_value[processname]=[x-y for x,y in zip(value[str(processname)+"-"+components[0]],value[str(processname)+"-"+components[1]])]
#print sum_value
#print "DONE!!"
# print len(sum_value)
# print len(timestamps[components[0]])
if key is not 'contention':
ax.plot(diff_value[processname], label = str(processname)+"-"+'-'.join(components))
else:
ax1.plot(diff_value[processname], label = str(processname)+"-"+'-'.join(components))
if seq:
if key is not 'contention':
leg = ax.legend(loc='upper left')
leg.get_frame().set_alpha(0.5)
else:
leg = ax1.legend(loc='upper left')
leg.get_frame().set_alpha(0.5)
if key =='contention':
#plot the drop in performance of each process:
perf_drop = compute_contention(diff_value)
for process, drop in perf_drop.items():
ax2.plot(drop, label="Drop in perf of "+str(process))
#change to a function later
avg_perf_drop = sum(drop)/len(drop)
f_handle= open(config.execution_time_dir+'/estimateddrop-'+process+'-'+
''.join([p if p is not process else '' for p,d in perf_drop.items()])+'.log','w+')
f_handle.write(str(avg_perf_drop))
f_handle.close()
leg=ax2.legend(loc= 'upper left')
leg.get_frame().set_alpha(0.5)
plt.savefig(seq+"/"+'+'.join(components))
plt.cla()
if len(op_sum) > 0:
for key, components in op_sum.items():
print components
#print [(value[component]) for component in components]
#print [len(value[component]) for component in components]
sum_value={}
if key =='contention' :
print "KEY: ", key
ax1 = plt.subplot(2,1,1)
ax2 = plt.subplot(2,1,2)
else:
ax = plt.subplot(1, 1, 1)
for processname in csv_file_handle:
sum_value[processname]=sum(map(numpy.array, [value[str(processname)+"-"+component] for component in components]))
#print sum_value
#print "DONE!!"
# print len(sum_value)
# print len(timestamps[components[0]])
if key is not 'contention':
ax.plot(sum_value[processname], label = str(processname)+"-"+'+'.join(components))
else:
ax1.plot(sum_value[processname], label = str(processname)+"-"+'+'.join(components))
if seq:
if key is not 'contention':
leg = ax.legend(loc='upper left')
leg.get_frame().set_alpha(0.5)
else:
leg = ax1.legend(loc='upper left')
leg.get_frame().set_alpha(0.5)
if key =='contention':
#plot the drop in performance of each process:
perf_drop = compute_contention(sum_value)
#print perf_drop
for process, drop in perf_drop.items():
ax2.plot(drop, label="Drop in perf of "+str(process))
#change to a function later
if len(drop)>0:
avg_perf_drop = sum(drop)/len(drop)
f_handle= open(config.execution_time_dir+'/estimateddrop-'+process+'-'+
''.join([p if p is not process else '' for p,d in perf_drop.items()])+'.log','w+')
f_handle.write(str(avg_perf_drop))
f_handle.close()
leg=ax2.legend(loc= 'upper left')
leg.get_frame().set_alpha(0.5)
elif key =='total_bandwidth':
plt.cla()
ax = plt.subplot(1, 1, 1)
total_bw = total_bandwidth(sum_value)
ax.plot(total_bw['total'], label = 'Total Bandwidth')
leg = ax.legend(loc='upper left')
leg.get_frame().set_alpha(0.5)
plt.savefig(seq+"/"+key+": "+'+'.join(components))
plt.cla()
if len(op_div) > 0:
ax = plt.subplot(1, 1, 1)
for components in op_div:
print components
for processname in csv_file_handle:
ax.plot([numpy.float64(x)/y for x,y in zip(value[str(processname)+"-"+components[0]],value[str(processname)+"-"+components[1]])], label= str(processname)+"-"+'/'.join(components))
if seq:
leg = ax.legend( loc='upper left')
leg.get_frame().set_alpha(0.5)
plt.savefig(seq+"/"+'_'.join(components))
plt.cla()
plt.xlabel('Time')
if val.find('%') >= 0:
plt.ylabel('Bottleneck %')
else:
plt.ylabel("Counter value")
if output:
plt.savefig(output)
else:
if not seq:
plt.show()
def plot(file, pstyle = 'ggplot', output=None, seq=None, xkcd=False):
# op_sum = {'1':['L1-dcache-loads','L1-dcache-stores','L1-dcache-prefetches','L1-icache-loads'],
# '2':['L1-dcache-load-misses','L1-dcache-store-misses','L1-dcache-prefetch-misses','L1-icache-load-misses'],
# '3':[ 'LLC-loads','LLC-stores','LLC-prefetches'],
# '4':['LLC-load-misses','LLC-store-misses','LLC-prefetch-misses'],
# '5':['dTLB-loads','dTLB-stores','iTLB-loads'],
# '6':['dTLB-load-misses','dTLB-store-misses','iTLB-load-misses'],
# 'Bandwidth':['offcore_response_corewb_local_dram_0','offcore_response_prefetch_any_llc_miss_0','LLC-prefetches','cache-misses']}
# op_div = [['cache-references','uops_retired_any'],['cache-misses','uops_retired_any'], ['instructions','cycles'],
# ['cache-misses','cache-references']]
#enable for i7
op_sum = {
'contention': ['cache-misses'],
'band': ['cache-references', 'cache-misses'],
'total_bandwidth': ['cache-references']
}
op_div= [['instructions','cycles'],['cache-misses','cache-references'],['cache-references','cycles'], ['cache-misses','cycles']]
print pstyle
if pstyle:
try:
from mpltools import style
style.use( pstyle)
except ImportError:
print "Need mpltools for setting styles (pip install mpltools)"
import gen_level
try:
import brewer2mpl
all_colors = brewer2mpl.get_map('Paired', 'Qualitative', 12).hex_colors
except ImportError:
print "Install brewer2mpl for better colors (pip install brewer2mpl)"
all_colors = ('green','orange','red','blue',
'black','olive','purple','#6960EC', '#F0FFFF',
'#728C00', '#827B60', '#F87217', '#E55451', # 16
'#F88017', '#C11B17', '#17BFC2', '#C48793') # 20
cur_colors = collections.defaultdict(lambda: all_colors)
assigned = dict()
if file:
try:
inf = open( file, "r")
except:
return
else:
inf = sys.stdin
rc = csv.reader(inf)
timestamps = dict()
value = dict()
val = ""
for r in rc:
if burst:
if len(r) == 2:
ts=0
val, event = r
else:
continue
if not burst:
# timestamp,event,value
if len(r) < 3:
continue
print r
if len(r) >= 5:
ts, event, val, thresh, desc = r
elif len(r) >= 4:
ts, val, unit, event = r
else:
ts, val, event = r
if event not in assigned:
level = gen_level.get_level(event)
assigned[event] = cur_colors[level][0]
cur_colors[level] = cur_colors[level][1:]
if len(cur_colors[level]) == 0:
cur_colors[level] = all_colors
value[event] = []
timestamps[event] = []
timestamps[event].append(float(ts))
try:
value[event].append(float(val.replace("%","")))
except ValueError:
value[event].append(0.0)
levels = dict()
for j in assigned.keys():
levels[gen_level.get_level(j)] = True
if xkcd:
try:
plt.xkcd()
except NameError:
print "Please update matplotlib. Cannot enable xkcd mode."
#print value
if normalize:
for key in value:
entries= value[key]
normalized_values = [numpy.float64(entry)/max(entries) for entry in entries]
value[key] = normalized_values
if seq:
os.umask(0000)
if os.path.exists(seq):
shutil.rmtree(seq)
os.makedirs(seq)
else:
os.makedirs(seq)
n = 1
print assigned.keys()
for l in levels.keys():
ax = plt.subplot(len(levels), 1, n)
if val.find('%') >= 0:
ax.set_ylim(0, 100)
t = []
for j in assigned.keys():
print j, gen_level.get_level(j), l
if gen_level.get_level(j) == l:
t.append(j)
if not burst:
if 'style' not in globals():
ax.plot(timestamps[j], value[j], assigned[j])
else:
ax.plot(timestamps[j], value[j])
else:
ax.plot(value[j])
if seq:
leg = ax.legend([j], loc='upper left')
leg.get_frame().set_alpha(0.5)
plt.savefig(seq+"/"+j)
plt.cla()
leg = ax.legend(t, loc='upper left')
leg.get_frame().set_alpha(0.5)
n += 1
if len(op_sum) > 0:
for key, components in op_sum.items():
print components
#print [(value[component]) for component in components]
#print [len(value[component]) for component in components]
sum_value=sum(map(numpy.array, [value[component] for component in components]))
#print sum_value
#print "DONE!!"
# print len(sum_value)
# print len(timestamps[components[0]])
if not burst:
ax.plot(timestamps[components[0]], sum_value)
else:
ax.plot(sum_value)
if seq:
leg = ax.legend(['+'.join(components)], loc='upper left')
leg.get_frame().set_alpha(0.5)
plt.savefig(seq+"/"+'+'.join(components))
plt.cla()
if len(op_div) > 0:
for components in op_div:
print components
if not burst:
ax.plot(timestamps[components[0]],[numpy.float64(x)/y for x,y in zip(value[components[0]],value[components[1]])])
else:
ax.plot([numpy.float64(x)/y for x,y in zip(value[components[0]],value[components[1]])])
if seq:
leg = ax.legend(['/'.join(components)], loc='upper left')
leg.get_frame().set_alpha(0.5)
plt.savefig(seq+"/"+'_'.join(components))
plt.cla()
plt.xlabel('Time')
if val.find('%') >= 0:
plt.ylabel('Bottleneck %')
else:
plt.ylabel("Counter value")
if output:
plt.savefig(output)
else:
if not seq:
plt.show()
if len(sys.argv) > 1:
inf = open(sys.argv[1], "r")
else:
inf = sys.stdin
rc = csv.reader(inf)
timestamps = dict()
value = dict()
for r in rc:
# timestamp,event,value
if len(r) < 3:
continue
event = r[1]
if event not in assigned:
level = gen_level.get_level(event)
print level, event
assigned[event] = cur_colors[level][0]
cur_colors[level] = cur_colors[level][1:]
if len(cur_colors[level]) == 0:
cur_colors[level] = all_colors
value[event] = []
timestamps[event] = []
timestamps[event].append(r[0])
try:
value[event].append(float(r[2].replace("%", "")))
except ValueError:
value[event].append(0.0)
levels = dict()
for j in assigned.keys():
ratios = defaultdict(list)
timestamps = []
rc = csv.reader(open(args.file, "r"))
ts = None
levels = defaultdict(list)
vals = None
for r in rc:
if len(r) < 4:
continue
if not re.match(r"\d+(\.\d*)", r[0]):
r = ["0.0"] + r
if r[3] == "metric":
l = METRIC_LEVEL # put at end
else:
l = gen_level.get_level(r[1])
if r[1] not in levels[l]:
levels[l].append(r[1])
t = math.trunc(float(r[0]) * 100) / 100.0
if t != ts:
timestamps.append(t)
ts = t
flush_vals(ratios, vals)
vals = dict()
vals[r[1]] = float(r[2].replace("%", ""))
flush_vals(ratios, vals)
if args.verbose:
print "time", len(timestamps), timestamps
for j in ratios.keys():
print j, ratios[j]