def plot_mmm(self,
ax,
index,
xscale=1.0,
yscale=1.0,
xlabel='',
ylabel='',
do_rate=False):
tmid = (self.ts.t[:-1] + self.ts.t[1:]) / 2.0
d = []
for k in self.ts.j.hosts.keys():
v = self.ts.assemble(index, k, 0)
if do_rate:
d.append(numpy.divide(numpy.diff(v), numpy.diff(self.ts.t)))
else:
d.append((v[:-1] + v[1:]) / 2.0)
a = numpy.array(d)
mn = []
p25 = []
p50 = []
p75 = []
mx = []
for i in range(len(self.ts.t) - 1):
mn.append(min(a[:, i]))
p25.append(score(a[:, i], 25))
p50.append(score(a[:, i], 50))
p75.append(score(a[:, i], 75))
mx.append(max(a[:, i]))
mn = numpy.array(mn)
p25 = numpy.array(p25)
p50 = numpy.array(p50)
p75 = numpy.array(p75)
mx = numpy.array(mx)
ax.hold = True
ax.plot(tmid / xscale, mn / yscale, '--')
ax.plot(tmid / xscale, p25 / yscale)
ax.plot(tmid / xscale, p50 / yscale)
ax.plot(tmid / xscale, p75 / yscale)
ax.plot(tmid / xscale, mx / yscale, '--')
self.setlabels(ax, index, xlabel, ylabel, yscale)
ax.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(nbins=4))
tspl_utils.adjust_yaxis_range(ax, 0.1)
def plot_lines(self,ax,index,xscale=1.0,yscale=1.0,xlabel='',ylabel='',
do_rate=True):
ax.hold=True
for k in self.ts.j.hosts.keys():
v=self.ts.assemble(index,k,0)
if do_rate:
val=numpy.divide(numpy.diff(v),numpy.diff(self.ts.t))
else:
val=(v[:-1]+v[1:])/(2.0)
ax.step(self.ts.t/xscale,numpy.append(val,[val[-1]])/yscale,where="post")
tspl_utils.adjust_yaxis_range(ax,0.1)
ax.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(nbins=6))
self.setlabels(ax,index,xlabel,ylabel,yscale)
ax.set_xlim([0,self.ts.t[-1]/3600.])
def plot_lines(self,ax,index,xscale=1.0,yscale=1.0,xlabel='',ylabel='',
do_rate=True):
tmid=(self.ts.t[:-1]+self.ts.t[1:])/2.0
ax.hold=True
for k in self.ts.j.hosts.keys():
v=self.ts.assemble(index,k,0)
if do_rate:
rate=numpy.divide(numpy.diff(v),numpy.diff(self.ts.t))
ax.plot(tmid/xscale,rate/yscale)
else:
val=(v[:-1]+v[1:])/2.0
ax.plot(tmid/xscale,val/yscale)
tspl_utils.adjust_yaxis_range(ax,0.1)
ax.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(nbins=6))
self.setlabels(ax,index,xlabel,ylabel,yscale)
def plot_mmm(self,ax,index,xscale=1.0,yscale=1.0,xlabel='',ylabel='',
do_rate=False):
tmid=(self.ts.t[:-1]+self.ts.t[1:])/2.0
d=[]
for k in self.ts.j.hosts.keys():
v=self.ts.assemble(index,k,0)
if do_rate:
d.append(numpy.divide(numpy.diff(v),numpy.diff(self.ts.t)))
else:
d.append((v[:-1]+v[1:])/2.0)
a=numpy.array(d)
mn=[]
p25=[]
p50=[]
p75=[]
mx=[]
for i in range(len(self.ts.t)-1):
mn.append(min(a[:,i]))
p25.append(score(a[:,i],25))
p50.append(score(a[:,i],50))
p75.append(score(a[:,i],75))
mx.append(max(a[:,i]))
mn=numpy.array(mn)
p25=numpy.array(p25)
p50=numpy.array(p50)
p75=numpy.array(p75)
mx=numpy.array(mx)
ax.hold=True
ax.plot(tmid/xscale,mn/yscale,'--')
ax.plot(tmid/xscale,p25/yscale)
ax.plot(tmid/xscale,p50/yscale)
ax.plot(tmid/xscale,p75/yscale)
ax.plot(tmid/xscale,mx/yscale,'--')
self.setlabels(ax,index,xlabel,ylabel,yscale)
ax.yaxis.set_major_locator( matplotlib.ticker.MaxNLocator(nbins=4))
tspl_utils.adjust_yaxis_range(ax,0.1)
def plot(self,jobid,job_data=None):
self.setup(jobid,job_data=job_data)
ts = self.ts
self.fig = Figure(figsize=(10,8),dpi=80)
self.ax=self.fig.add_subplot(1,1,1)
self.ax=[self.ax]
self.fig.subplots_adjust(hspace=0.35)
markers = ('o','x','+','^','s','8','p',
'h','*','D','<','>','v','d','.')
colors = ('b','g','r','c','m','k','y')
tmid=(self.ts.t[:-1]+self.ts.t[1:])/2.0
cnt=0
for v in ts.data:
for host in v:
for vals in v[host]:
rate=numpy.diff(vals)/numpy.diff(ts.t)
c=colors[cnt % len(colors)]
m=markers[cnt % len(markers)]
self.ax[0].plot(tmid/3600., rate, marker=m,
markeredgecolor=c, linestyle='-', color=c,
markerfacecolor='None', label=self.k2[cnt])
self.ax[0].hold=True
cnt=cnt+1
self.ax[0].set_ylabel('Meta Data Rate (op/s)')
tspl_utils.adjust_yaxis_range(self.ax[0],0.1)
handles,labels=self.ax[0].get_legend_handles_labels()
new_handles={}
for h,l in zip(handles,labels):
new_handles[l]=h
box = self.ax[0].get_position()
self.ax[0].set_position([box.x0, box.y0, box.width * 0.9, box.height])
self.ax[0].legend(new_handles.values(),new_handles.keys(),prop={'size':8},
bbox_to_anchor=(1.05,1), borderaxespad=0., loc=2)
self.output('metadata')
def plot_lines(self,
ax,
index,
xscale=1.0,
yscale=1.0,
xlabel='',
ylabel='',
do_rate=True):
ax.hold = True
for k in self.ts.j.hosts.keys():
v = self.ts.assemble(index, k, 0)
if do_rate:
val = numpy.divide(numpy.diff(v), numpy.diff(self.ts.t))
else:
val = (v[:-1] + v[1:]) / (2.0)
ax.step(self.ts.t / xscale,
numpy.append(val, [val[-1]]) / yscale,
where="post")
tspl_utils.adjust_yaxis_range(ax, 0.1)
ax.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(nbins=6))
self.setlabels(ax, index, xlabel, ylabel, yscale)
ax.set_xlim([0, self.ts.t[-1] / 3600.])
def plot(self,jobid,job_data=None):
if not self.setup(jobid,job_data=job_data): return
wayness=self.ts.wayness
if self.lariat_data != 'pass':
if self.lariat_data.wayness and self.lariat_data.wayness < self.ts.wayness:
wayness=self.lariat_data.wayness
if self.wide:
self.fig = Figure(figsize=(15.5,12),dpi=110)
self.ax=self.fig.add_subplot(6,2,2)
cols = 2
shift = 2
else:
self.fig = Figure(figsize=(8,12),dpi=110)
self.ax=self.fig.add_subplot(6,1,1)
cols = 1
shift = 1
if self.mode == 'hist':
plot=self.plot_thist
elif self.mode == 'percentile':
plot=self.plot_mmm
else:
plot=self.plot_lines
k1_tmp=self.k1[self.ts.pmc_type]
k2_tmp=self.k2[self.ts.pmc_type]
# Plot key 1 for flops
ax = self.fig.add_subplot(6,cols,1*shift)
schema = self.ts.j.get_schema(self.ts.pmc_type)
for host_name in self.ts.j.hosts.keys():
stats = self.ts.j.aggregate_stats(self.ts.pmc_type,host_names=[host_name])
if self.ts.pmc_type == 'intel_snb' :
if 'SSE_D_ALL' in schema:
flops = stats[0][:,schema['SSE_D_ALL'].index]+4*stats[0][:,schema['SIMD_D_256'].index]
elif 'SSE_DOUBLE_SCALAR' in schema:
flops = stats[0][:,schema['SSE_DOUBLE_SCALAR'].index]+2*stats[0][:,schema['SSE_DOUBLE_PACKED'].index]+4*stats[0][:,schema['SIMD_DOUBLE_256'].index]
else: print("FLOP stats not available for JOBID",self.ts.j.id)
elif self.ts.pmc_type == 'intel_pmc3' or self.ts.pmc_type == 'intel_nhm' or self.ts.pmc_type == 'intel_wtm':
if 'FP_COMP_OPS_EXE_SSE_PACKED' in schema and 'FP_COMP_OPS_EXE_SSE_SCALAR' in schema:
flops = 2*stats[0][:,schema['FP_COMP_OPS_EXE_SSE_PACKED'].index]+stats[0][:,schema['FP_COMP_OPS_EXE_SSE_SCALAR'].index]
else: print("FLOP stats not available for JOBID",self.ts.j.id)
elif self.ts.pmc_type == 'intel_hsw' :
print('Haswell does not support FLOP counters')
else:
print(self.ts.pmc_type + ' not currently supported')
continue
try:
flops = numpy.diff(flops)/numpy.diff(self.ts.t)/1.0e9
ax.step(self.ts.t/3600., numpy.append(flops,[flops[-1]]), where="post")
ax.set_ylabel('Dbl GFLOPS')
ax.set_xlim([0.,self.ts.t[-1]/3600.])
tspl_utils.adjust_yaxis_range(ax,0.1)
except: print("FLOP plot not available for JOBID",self.ts.j.id)
# Plot key 2
if self.ts.pmc_type == 'intel_snb' or self.ts.pmc_type == 'intel_hsw':
idx0=k2_tmp.index('CAS_READS')
idx1=k2_tmp.index('CAS_WRITES')
if self.ts.pmc_type == 'intel_pmc3' or self.ts.pmc_type == 'intel_nhm' or self.ts.pmc_type == 'intel_wtm' :
idx0=k2_tmp.index('MEM_UNCORE_RETIRED_REMOTE_DRAM')
idx1=k2_tmp.index('MEM_UNCORE_RETIRED_LOCAL_DRAM')
try:
plot(self.fig.add_subplot(6,cols,2*shift), [idx0,idx1], 3600., 1.0/64.0*1024.*1024.*1024., ylabel='Total Mem BW GB/s')
except:
pass
#Plot key 3
idx0=k2_tmp.index('MemUsed')
idx1=k2_tmp.index('FilePages')
idx2=k2_tmp.index('Slab')
plot(self.fig.add_subplot(6,cols,3*shift), [idx0,-idx1,-idx2], 3600.,2.**30.0, ylabel='Memory Usage GB',do_rate=False)
# Plot lnet sum rate
idx0=k1_tmp.index('lnet')
idx1=idx0 + k1_tmp[idx0+1:].index('lnet') + 1
plot(self.fig.add_subplot(6,cols,4*shift), [idx0,idx1], 3600., 1024.**2, ylabel='Total lnet MB/s')
# Plot remaining IB sum rate
if self.ts.pmc_type == 'intel_snb' or self.ts.pmc_type == 'intel_hsw':
idx2=k1_tmp.index('ib_sw')
idx3=idx2 + k1_tmp[idx2+1:].index('ib_sw') + 1
if self.ts.pmc_type == 'intel_pmc3' or self.ts.pmc_type == 'intel_nhm' or self.ts.pmc_type == 'intel_wtm':
idx2=k1_tmp.index('ib_ext')
idx3=idx2 + k1_tmp[idx2+1:].index('ib_ext') + 1
plot(self.fig.add_subplot(6,cols,5*shift),[idx2,idx3,-idx0,-idx1],3600.,2.**20,
ylabel='Total (ib-lnet) MB/s')
#Plot CPU user time
idx0=k2_tmp.index('user')
plot(self.fig.add_subplot(6,cols,6*shift),[idx0],3600.,wayness*100.,
xlabel='Time (hr)',
ylabel='Total cpu user\nfraction')
self.fig.subplots_adjust(hspace=0.35)
self.output('master')
def plot(self,jobid,job_data=None):
if not self.setup(jobid,job_data=job_data): return
if self.wide:
self.fig = Figure(figsize=(15.5,12),dpi=110)
self.ax=self.fig.add_subplot(6,2,2)
cols = 2
shift = 2
else:
self.fig = Figure(figsize=(8,12),dpi=110)
self.ax=self.fig.add_subplot(6,1,1)
cols = 1
shift = 1
if self.mode == 'hist':
plot=self.plot_thist
elif self.mode == 'percentile':
plot=self.plot_mmm
else:
plot=self.plot_lines
k1_tmp=self.k1[self.ts.pmc_type]
k2_tmp=self.k2[self.ts.pmc_type]
processor_schema = self.ts.j.schemas[self.ts.pmc_type]
# Plot key 1 for flops
plot_ctr = 0
try:
if 'SSE_D_ALL' in processor_schema and 'SIMD_D_256' in processor_schema:
idx0 = k2_tmp.index('SSE_D_ALL')
idx1 = None
idx2 = k2_tmp.index('SIMD_D_256')
elif 'SSE_DOUBLE_SCALAR' in processor_schema and 'SSE_DOUBLE_PACKED' in processor_schema and 'SIMD_DOUBLE_256' in processor_schema:
idx0 = k2_tmp.index('SSE_DOUBLE_SCALAR')
idx1 = k2_tmp.index('SSE_DOUBLE_PACKED')
idx2 = k2_tmp.index('SIMD_DOUBLE_256')
elif 'FP_COMP_OPS_EXE_SSE_PACKED' in processor_schema and 'FP_COMP_OPS_EXE_SSE_SCALAR' in processor_schema:
idx0 = k2_tmp.index('FP_COMP_OPS_EXE_SSE_SCALAR')
idx1 = k2_tmp.index('FP_COMP_OPS_EXE_SSE_PACKED')
idx2 = None
else:
print("FLOP stats not available for JOBID",self.ts.j.id)
raise
plot_ctr += 1
ax = self.fig.add_subplot(6,cols,plot_ctr*shift)
for host_name in self.ts.j.hosts.keys():
flops = self.ts.assemble([idx0],host_name,0)
if idx1: flops += 2*self.ts.assemble([idx1],host_name,0)
if idx2: flops += 4*self.ts.assemble([idx2],host_name,0)
flops = numpy.diff(flops)/numpy.diff(self.ts.t)/1.0e9
ax.step(self.ts.t/3600., numpy.append(flops, [flops[-1]]),
where="post")
ax.set_ylabel('Dbl GFLOPS')
ax.set_xlim([0.,self.ts.t[-1]/3600.])
tspl_utils.adjust_yaxis_range(ax,0.1)
except:
print sys.exc_info()[0]
print("FLOP plot not available for JOBID",self.ts.j.id)
# Plot key 2
try:
if 'CAS_READS' in k2_tmp and 'CAS_WRITES' in k2_tmp:
idx0=k2_tmp.index('CAS_READS')
idx1=k2_tmp.index('CAS_WRITES')
elif 'MEM_UNCORE_RETIRED_REMOTE_DRAM' in k2_tmp and 'MEM_UNCORE_RETIRED_LOCAL_DRAM' in k2_tmp:
idx0=k2_tmp.index('MEM_UNCORE_RETIRED_REMOTE_DRAM')
idx1=k2_tmp.index('MEM_UNCORE_RETIRED_LOCAL_DRAM')
else:
print(self.ts.pmc_type + ' missing Memory Bandwidth data' + ' for jobid ' + self.ts.j.id )
raise
plot_ctr += 1
plot(self.fig.add_subplot(6,cols,plot_ctr*shift), [idx0,idx1], 3600.,
1.0/64.0*1024.*1024.*1024., ylabel='Total Mem BW GB/s')
except:
print(self.ts.pmc_type + ' missing Memory Bandwidth plot' + ' for jobid ' + self.ts.j.id )
#Plot key 3
idx0=k2_tmp.index('MemUsed')
idx1=k2_tmp.index('FilePages')
idx2=k2_tmp.index('Slab')
plot_ctr += 1
plot(self.fig.add_subplot(6,cols,plot_ctr*shift), [idx0,-idx1,-idx2], 3600.,2.**30.0,
ylabel='Memory Usage GB',do_rate=False)
# Plot lnet sum rate
idx0=k1_tmp.index('lnet')
idx1=idx0 + k1_tmp[idx0+1:].index('lnet') + 1
plot_ctr += 1
plot(self.fig.add_subplot(6,cols,plot_ctr*shift), [idx0,idx1], 3600., 1024.**2, ylabel='Total lnet MB/s')
# Plot remaining IB sum rate
if 'ib_ext' in self.ts.j.hosts.values()[0].stats:
try:
idx2=k1_tmp.index('ib_sw')
idx3=idx2 + k1_tmp[idx2+1:].index('ib_sw') + 1
except:
idx2=k1_tmp.index('ib_ext')
idx3=idx2 + k1_tmp[idx2+1:].index('ib_ext') + 1
try:
plot_ctr += 1
plot(self.fig.add_subplot(6,cols,plot_ctr*shift),[idx2,idx3,-idx0,-idx1],3600.,2.**20,
ylabel='Total (ib-lnet) MB/s')
except: pass
#Plot CPU user time
idx0 = [k2_tmp.index('user')]
plot_ctr += 1
wayness = len(self.ts.j.hosts.values()[0].stats['cpu'].keys())
plot(self.fig.add_subplot(6,cols,plot_ctr*shift),idx0,3600.,wayness,
xlabel='Time (hr)',
ylabel='cpu usage %')
self.fig.subplots_adjust(hspace=0.35)
self.output('master')