def plot_mmm(self, ax, index, xscale=1.0, yscale=1.0, xlabel='', ylabel='', do_rate=False): tmid = (self.ts.t[:-1] + self.ts.t[1:]) / 2.0 d = [] for k in self.ts.j.hosts.keys(): v = self.ts.assemble(index, k, 0) if do_rate: d.append(numpy.divide(numpy.diff(v), numpy.diff(self.ts.t))) else: d.append((v[:-1] + v[1:]) / 2.0) a = numpy.array(d) mn = [] p25 = [] p50 = [] p75 = [] mx = [] for i in range(len(self.ts.t) - 1): mn.append(min(a[:, i])) p25.append(score(a[:, i], 25)) p50.append(score(a[:, i], 50)) p75.append(score(a[:, i], 75)) mx.append(max(a[:, i])) mn = numpy.array(mn) p25 = numpy.array(p25) p50 = numpy.array(p50) p75 = numpy.array(p75) mx = numpy.array(mx) ax.hold = True ax.plot(tmid / xscale, mn / yscale, '--') ax.plot(tmid / xscale, p25 / yscale) ax.plot(tmid / xscale, p50 / yscale) ax.plot(tmid / xscale, p75 / yscale) ax.plot(tmid / xscale, mx / yscale, '--') self.setlabels(ax, index, xlabel, ylabel, yscale) ax.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(nbins=4)) tspl_utils.adjust_yaxis_range(ax, 0.1)
def plot_lines(self,ax,index,xscale=1.0,yscale=1.0,xlabel='',ylabel='', do_rate=True): ax.hold=True for k in self.ts.j.hosts.keys(): v=self.ts.assemble(index,k,0) if do_rate: val=numpy.divide(numpy.diff(v),numpy.diff(self.ts.t)) else: val=(v[:-1]+v[1:])/(2.0) ax.step(self.ts.t/xscale,numpy.append(val,[val[-1]])/yscale,where="post") tspl_utils.adjust_yaxis_range(ax,0.1) ax.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(nbins=6)) self.setlabels(ax,index,xlabel,ylabel,yscale) ax.set_xlim([0,self.ts.t[-1]/3600.])
def plot_lines(self,ax,index,xscale=1.0,yscale=1.0,xlabel='',ylabel='', do_rate=True): tmid=(self.ts.t[:-1]+self.ts.t[1:])/2.0 ax.hold=True for k in self.ts.j.hosts.keys(): v=self.ts.assemble(index,k,0) if do_rate: rate=numpy.divide(numpy.diff(v),numpy.diff(self.ts.t)) ax.plot(tmid/xscale,rate/yscale) else: val=(v[:-1]+v[1:])/2.0 ax.plot(tmid/xscale,val/yscale) tspl_utils.adjust_yaxis_range(ax,0.1) ax.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(nbins=6)) self.setlabels(ax,index,xlabel,ylabel,yscale)
def plot_mmm(self,ax,index,xscale=1.0,yscale=1.0,xlabel='',ylabel='', do_rate=False): tmid=(self.ts.t[:-1]+self.ts.t[1:])/2.0 d=[] for k in self.ts.j.hosts.keys(): v=self.ts.assemble(index,k,0) if do_rate: d.append(numpy.divide(numpy.diff(v),numpy.diff(self.ts.t))) else: d.append((v[:-1]+v[1:])/2.0) a=numpy.array(d) mn=[] p25=[] p50=[] p75=[] mx=[] for i in range(len(self.ts.t)-1): mn.append(min(a[:,i])) p25.append(score(a[:,i],25)) p50.append(score(a[:,i],50)) p75.append(score(a[:,i],75)) mx.append(max(a[:,i])) mn=numpy.array(mn) p25=numpy.array(p25) p50=numpy.array(p50) p75=numpy.array(p75) mx=numpy.array(mx) ax.hold=True ax.plot(tmid/xscale,mn/yscale,'--') ax.plot(tmid/xscale,p25/yscale) ax.plot(tmid/xscale,p50/yscale) ax.plot(tmid/xscale,p75/yscale) ax.plot(tmid/xscale,mx/yscale,'--') self.setlabels(ax,index,xlabel,ylabel,yscale) ax.yaxis.set_major_locator( matplotlib.ticker.MaxNLocator(nbins=4)) tspl_utils.adjust_yaxis_range(ax,0.1)
def plot(self,jobid,job_data=None): self.setup(jobid,job_data=job_data) ts = self.ts self.fig = Figure(figsize=(10,8),dpi=80) self.ax=self.fig.add_subplot(1,1,1) self.ax=[self.ax] self.fig.subplots_adjust(hspace=0.35) markers = ('o','x','+','^','s','8','p', 'h','*','D','<','>','v','d','.') colors = ('b','g','r','c','m','k','y') tmid=(self.ts.t[:-1]+self.ts.t[1:])/2.0 cnt=0 for v in ts.data: for host in v: for vals in v[host]: rate=numpy.diff(vals)/numpy.diff(ts.t) c=colors[cnt % len(colors)] m=markers[cnt % len(markers)] self.ax[0].plot(tmid/3600., rate, marker=m, markeredgecolor=c, linestyle='-', color=c, markerfacecolor='None', label=self.k2[cnt]) self.ax[0].hold=True cnt=cnt+1 self.ax[0].set_ylabel('Meta Data Rate (op/s)') tspl_utils.adjust_yaxis_range(self.ax[0],0.1) handles,labels=self.ax[0].get_legend_handles_labels() new_handles={} for h,l in zip(handles,labels): new_handles[l]=h box = self.ax[0].get_position() self.ax[0].set_position([box.x0, box.y0, box.width * 0.9, box.height]) self.ax[0].legend(new_handles.values(),new_handles.keys(),prop={'size':8}, bbox_to_anchor=(1.05,1), borderaxespad=0., loc=2) self.output('metadata')
def plot_lines(self, ax, index, xscale=1.0, yscale=1.0, xlabel='', ylabel='', do_rate=True): ax.hold = True for k in self.ts.j.hosts.keys(): v = self.ts.assemble(index, k, 0) if do_rate: val = numpy.divide(numpy.diff(v), numpy.diff(self.ts.t)) else: val = (v[:-1] + v[1:]) / (2.0) ax.step(self.ts.t / xscale, numpy.append(val, [val[-1]]) / yscale, where="post") tspl_utils.adjust_yaxis_range(ax, 0.1) ax.yaxis.set_major_locator(matplotlib.ticker.MaxNLocator(nbins=6)) self.setlabels(ax, index, xlabel, ylabel, yscale) ax.set_xlim([0, self.ts.t[-1] / 3600.])
def plot(self,jobid,job_data=None): if not self.setup(jobid,job_data=job_data): return wayness=self.ts.wayness if self.lariat_data != 'pass': if self.lariat_data.wayness and self.lariat_data.wayness < self.ts.wayness: wayness=self.lariat_data.wayness if self.wide: self.fig = Figure(figsize=(15.5,12),dpi=110) self.ax=self.fig.add_subplot(6,2,2) cols = 2 shift = 2 else: self.fig = Figure(figsize=(8,12),dpi=110) self.ax=self.fig.add_subplot(6,1,1) cols = 1 shift = 1 if self.mode == 'hist': plot=self.plot_thist elif self.mode == 'percentile': plot=self.plot_mmm else: plot=self.plot_lines k1_tmp=self.k1[self.ts.pmc_type] k2_tmp=self.k2[self.ts.pmc_type] # Plot key 1 for flops ax = self.fig.add_subplot(6,cols,1*shift) schema = self.ts.j.get_schema(self.ts.pmc_type) for host_name in self.ts.j.hosts.keys(): stats = self.ts.j.aggregate_stats(self.ts.pmc_type,host_names=[host_name]) if self.ts.pmc_type == 'intel_snb' : if 'SSE_D_ALL' in schema: flops = stats[0][:,schema['SSE_D_ALL'].index]+4*stats[0][:,schema['SIMD_D_256'].index] elif 'SSE_DOUBLE_SCALAR' in schema: flops = stats[0][:,schema['SSE_DOUBLE_SCALAR'].index]+2*stats[0][:,schema['SSE_DOUBLE_PACKED'].index]+4*stats[0][:,schema['SIMD_DOUBLE_256'].index] else: print("FLOP stats not available for JOBID",self.ts.j.id) elif self.ts.pmc_type == 'intel_pmc3' or self.ts.pmc_type == 'intel_nhm' or self.ts.pmc_type == 'intel_wtm': if 'FP_COMP_OPS_EXE_SSE_PACKED' in schema and 'FP_COMP_OPS_EXE_SSE_SCALAR' in schema: flops = 2*stats[0][:,schema['FP_COMP_OPS_EXE_SSE_PACKED'].index]+stats[0][:,schema['FP_COMP_OPS_EXE_SSE_SCALAR'].index] else: print("FLOP stats not available for JOBID",self.ts.j.id) elif self.ts.pmc_type == 'intel_hsw' : print('Haswell does not support FLOP counters') else: print(self.ts.pmc_type + ' not currently supported') continue try: flops = numpy.diff(flops)/numpy.diff(self.ts.t)/1.0e9 ax.step(self.ts.t/3600., numpy.append(flops,[flops[-1]]), where="post") ax.set_ylabel('Dbl GFLOPS') ax.set_xlim([0.,self.ts.t[-1]/3600.]) tspl_utils.adjust_yaxis_range(ax,0.1) except: print("FLOP plot not available for JOBID",self.ts.j.id) # Plot key 2 if self.ts.pmc_type == 'intel_snb' or self.ts.pmc_type == 'intel_hsw': idx0=k2_tmp.index('CAS_READS') idx1=k2_tmp.index('CAS_WRITES') if self.ts.pmc_type == 'intel_pmc3' or self.ts.pmc_type == 'intel_nhm' or self.ts.pmc_type == 'intel_wtm' : idx0=k2_tmp.index('MEM_UNCORE_RETIRED_REMOTE_DRAM') idx1=k2_tmp.index('MEM_UNCORE_RETIRED_LOCAL_DRAM') try: plot(self.fig.add_subplot(6,cols,2*shift), [idx0,idx1], 3600., 1.0/64.0*1024.*1024.*1024., ylabel='Total Mem BW GB/s') except: pass #Plot key 3 idx0=k2_tmp.index('MemUsed') idx1=k2_tmp.index('FilePages') idx2=k2_tmp.index('Slab') plot(self.fig.add_subplot(6,cols,3*shift), [idx0,-idx1,-idx2], 3600.,2.**30.0, ylabel='Memory Usage GB',do_rate=False) # Plot lnet sum rate idx0=k1_tmp.index('lnet') idx1=idx0 + k1_tmp[idx0+1:].index('lnet') + 1 plot(self.fig.add_subplot(6,cols,4*shift), [idx0,idx1], 3600., 1024.**2, ylabel='Total lnet MB/s') # Plot remaining IB sum rate if self.ts.pmc_type == 'intel_snb' or self.ts.pmc_type == 'intel_hsw': idx2=k1_tmp.index('ib_sw') idx3=idx2 + k1_tmp[idx2+1:].index('ib_sw') + 1 if self.ts.pmc_type == 'intel_pmc3' or self.ts.pmc_type == 'intel_nhm' or self.ts.pmc_type == 'intel_wtm': idx2=k1_tmp.index('ib_ext') idx3=idx2 + k1_tmp[idx2+1:].index('ib_ext') + 1 plot(self.fig.add_subplot(6,cols,5*shift),[idx2,idx3,-idx0,-idx1],3600.,2.**20, ylabel='Total (ib-lnet) MB/s') #Plot CPU user time idx0=k2_tmp.index('user') plot(self.fig.add_subplot(6,cols,6*shift),[idx0],3600.,wayness*100., xlabel='Time (hr)', ylabel='Total cpu user\nfraction') self.fig.subplots_adjust(hspace=0.35) self.output('master')
def plot(self,jobid,job_data=None): if not self.setup(jobid,job_data=job_data): return if self.wide: self.fig = Figure(figsize=(15.5,12),dpi=110) self.ax=self.fig.add_subplot(6,2,2) cols = 2 shift = 2 else: self.fig = Figure(figsize=(8,12),dpi=110) self.ax=self.fig.add_subplot(6,1,1) cols = 1 shift = 1 if self.mode == 'hist': plot=self.plot_thist elif self.mode == 'percentile': plot=self.plot_mmm else: plot=self.plot_lines k1_tmp=self.k1[self.ts.pmc_type] k2_tmp=self.k2[self.ts.pmc_type] processor_schema = self.ts.j.schemas[self.ts.pmc_type] # Plot key 1 for flops plot_ctr = 0 try: if 'SSE_D_ALL' in processor_schema and 'SIMD_D_256' in processor_schema: idx0 = k2_tmp.index('SSE_D_ALL') idx1 = None idx2 = k2_tmp.index('SIMD_D_256') elif 'SSE_DOUBLE_SCALAR' in processor_schema and 'SSE_DOUBLE_PACKED' in processor_schema and 'SIMD_DOUBLE_256' in processor_schema: idx0 = k2_tmp.index('SSE_DOUBLE_SCALAR') idx1 = k2_tmp.index('SSE_DOUBLE_PACKED') idx2 = k2_tmp.index('SIMD_DOUBLE_256') elif 'FP_COMP_OPS_EXE_SSE_PACKED' in processor_schema and 'FP_COMP_OPS_EXE_SSE_SCALAR' in processor_schema: idx0 = k2_tmp.index('FP_COMP_OPS_EXE_SSE_SCALAR') idx1 = k2_tmp.index('FP_COMP_OPS_EXE_SSE_PACKED') idx2 = None else: print("FLOP stats not available for JOBID",self.ts.j.id) raise plot_ctr += 1 ax = self.fig.add_subplot(6,cols,plot_ctr*shift) for host_name in self.ts.j.hosts.keys(): flops = self.ts.assemble([idx0],host_name,0) if idx1: flops += 2*self.ts.assemble([idx1],host_name,0) if idx2: flops += 4*self.ts.assemble([idx2],host_name,0) flops = numpy.diff(flops)/numpy.diff(self.ts.t)/1.0e9 ax.step(self.ts.t/3600., numpy.append(flops, [flops[-1]]), where="post") ax.set_ylabel('Dbl GFLOPS') ax.set_xlim([0.,self.ts.t[-1]/3600.]) tspl_utils.adjust_yaxis_range(ax,0.1) except: print sys.exc_info()[0] print("FLOP plot not available for JOBID",self.ts.j.id) # Plot key 2 try: if 'CAS_READS' in k2_tmp and 'CAS_WRITES' in k2_tmp: idx0=k2_tmp.index('CAS_READS') idx1=k2_tmp.index('CAS_WRITES') elif 'MEM_UNCORE_RETIRED_REMOTE_DRAM' in k2_tmp and 'MEM_UNCORE_RETIRED_LOCAL_DRAM' in k2_tmp: idx0=k2_tmp.index('MEM_UNCORE_RETIRED_REMOTE_DRAM') idx1=k2_tmp.index('MEM_UNCORE_RETIRED_LOCAL_DRAM') else: print(self.ts.pmc_type + ' missing Memory Bandwidth data' + ' for jobid ' + self.ts.j.id ) raise plot_ctr += 1 plot(self.fig.add_subplot(6,cols,plot_ctr*shift), [idx0,idx1], 3600., 1.0/64.0*1024.*1024.*1024., ylabel='Total Mem BW GB/s') except: print(self.ts.pmc_type + ' missing Memory Bandwidth plot' + ' for jobid ' + self.ts.j.id ) #Plot key 3 idx0=k2_tmp.index('MemUsed') idx1=k2_tmp.index('FilePages') idx2=k2_tmp.index('Slab') plot_ctr += 1 plot(self.fig.add_subplot(6,cols,plot_ctr*shift), [idx0,-idx1,-idx2], 3600.,2.**30.0, ylabel='Memory Usage GB',do_rate=False) # Plot lnet sum rate idx0=k1_tmp.index('lnet') idx1=idx0 + k1_tmp[idx0+1:].index('lnet') + 1 plot_ctr += 1 plot(self.fig.add_subplot(6,cols,plot_ctr*shift), [idx0,idx1], 3600., 1024.**2, ylabel='Total lnet MB/s') # Plot remaining IB sum rate if 'ib_ext' in self.ts.j.hosts.values()[0].stats: try: idx2=k1_tmp.index('ib_sw') idx3=idx2 + k1_tmp[idx2+1:].index('ib_sw') + 1 except: idx2=k1_tmp.index('ib_ext') idx3=idx2 + k1_tmp[idx2+1:].index('ib_ext') + 1 try: plot_ctr += 1 plot(self.fig.add_subplot(6,cols,plot_ctr*shift),[idx2,idx3,-idx0,-idx1],3600.,2.**20, ylabel='Total (ib-lnet) MB/s') except: pass #Plot CPU user time idx0 = [k2_tmp.index('user')] plot_ctr += 1 wayness = len(self.ts.j.hosts.values()[0].stats['cpu'].keys()) plot(self.fig.add_subplot(6,cols,plot_ctr*shift),idx0,3600.,wayness, xlabel='Time (hr)', ylabel='cpu usage %') self.fig.subplots_adjust(hspace=0.35) self.output('master')