def show_metadata(args, metadata): """ It would be nice if I could unify this with Graph.mds_plot(). """ fig = plt.figure() ax = fig.add_subplot(111) steps = metadata.Steps.Steps values = metadata.MDS.Values Graph.timeSeries(ax, steps, values, 'b', label='ops', Ave=True) plt.xlabel('time') plt.ylabel(r'$ops/sec$') if not metadata.CPU is None: values = metadata.CPU.Values (handles, labels) = Graph.percent(ax, steps, values, 'k', label='% CPU', Ave=True) plt.legend(handles, labels) else: plt.legend() plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(metadata.begin.sie)) plt.title("%s %s metadata" % (dayStr, metadata.name)) page = StringIO.StringIO() plt.savefig(page, format='png') page.seek(0) plt.cla() return(page)
def do_plot(mds, plot, ymax, extra=None): fig = plt.figure() ax = fig.add_subplot(111) steps = mds.Steps.Steps values = mds.MDS.Values if ymax is None: ymax = np.max(values) Graph.timeSeries(ax, steps, values, 'b', label='MDS', Ave=True) plt.xlabel('time') plt.ylabel(r'$ops/sec$') if not mds.CPU is None: values = mds.CPU.Values (handles, labels) = Graph.percent(ax, steps, values, 'k', label='% CPU', Ave=True) if (not handles is None) and (not labels is None): plt.legend(handles, labels) else: print "mds.do_plot(): Warning - Plotting CPU utilization failed." else: plt.legend() plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') start_time = steps[0]/(24.0*60.0*60.0) + mpl.dates.date2num(datetime.date(1970,1,1)) plt.title("%s metadata operations for %s" % (mds.name, mpl.dates.num2date(start_time).strftime("%Y-%m-%d")) ) if ymax is None: ymax = ymax ax.set_ybound(lower=0, upper=ymax) if plot is None: plt.show() else: plt.savefig(plot) plt.cla()
def show_bulk(args, bulk): """ It would be nice if I could unify this with Graph.bulk_plot().That would entail using the StringIO function there and handling the save/show option in the caller. """ scale = 1024.0*1024.0 fig = plt.figure() ax = fig.add_subplot(111) steps = bulk.Steps.Steps ybound = 50000.0 values = bulk.Read.Values/scale Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True) values = bulk.Write.Values/scale Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True) plt.xlabel('time') plt.ylabel(r'$MiB/sec$') if not bulk.CPU is None: values = bulk.CPU.Values (handles, labels) = Graph.percent(ax, steps, values, 'k', label='% CPU', Ave=True) plt.legend(handles, labels) else: plt.legend() plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(bulk.begin.sie)) plt.title("%s %s aggregate I/O" % (dayStr, bulk.name)) ax.set_ybound(lower = 0, upper = ybound) page = StringIO.StringIO() plt.savefig(page, format='png') page.seek(0) plt.cla() return(page)
def time_plot_all_bins(args, BO): if not args.stat in BO.BrwNameDict: print "brwost.py.do_action(): %s not found in the set of Statistics" % args.stat return Bins = BO.Bins[BO.BrwNameDict[args.stat]] fig = plt.figure() ax = fig.add_subplot(111) steps = BO.Steps.Steps times = np.diff(steps) ymax = 0 bin_num = 0 formats = ['-', '-', '-', '-', '-', '-', '-', '--', '--'] colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'b', 'r'] labels = ['4k', '8k', '16k', '32k', '64k', '128k', '256k', '512k', '1m'] for bin in Bins.Bins: if args.read == True: io = "read" read = BO.Read[Bins.id] values = read.Values[Bins.BinDict[bin],:] values[1:] = np.diff(values)/times values[0] = 0.0 max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, color=colors[bin_num], label=labels[bin_num], format=formats[bin_num]) else: # This ends up doing write I/O by default, you have to request reads io = "write" write = BO.Write[Bins.id] values = write.Values[Bins.BinDict[args.bin],:] values[1:] = np.diff(values)/times values[0] = 0.0 max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, color=colors[bin_num], label=labels[bin_num], format=formats[bin_num]) bin_num += 1 plt.xlabel('time') plt.ylabel(r'$count/sec$') plt.legend() plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(BO.begin.sie)) plt.title("%s %s %s RPC %s stats" % (dayStr, BO.name, args.stat, io)) if (not args.ybound is None) and (args.ybound > ymax): ymax = args.ybound ax.set_ybound(lower = 0, upper = ymax) if args.plot is None: plt.show() else: plt.savefig(args.plot) plt.cla() return
def doAction(args, b_sie, e_sie, day, fs, host, darFile, scratchFile, scratch2File): darSteps, darRead, darWrite = ratesfromh5dar(args, b_sie, e_sie, darFile) scratchSteps, scratchRead, scratchWrite = ratesfromh5lmt(args, b_sie, e_sie, scratchFile) if len(darSteps) != len(scratchSteps): print "len(darSteps) = %d" % len(darSteps) print "len(scratchSteps) = %d" % len(scratchSteps) print "len(darRead) = %d" % len(darRead) print "len(scratchRead) = %d" % len(scratchRead) print "len(darWrite) = %d" % len(darWrite) print "len(scratchWrite) = %d" % len(scratchWrite) return if darSteps[0] != scratchSteps[0]: print "darSteps[0] = %d" % darSteps[0] print "scratchSteps[0] = %d" % scratchSteps[0] return scratch2Steps, scratch2Read, scratch2Write = ratesfromh5lmt(args, b_sie, e_sie, scratch2File) if len(darSteps) != len(scratch2Steps): print "len(darSteps) = %d" % len(darSteps) print "len(scratch2Steps) = %d" % len(scratch2Steps) print "len(darRead) = %d" % len(darRead) print "len(scratch2Read) = %d" % len(scratch2Read) print "len(darWrite) = %d" % len(darWrite) print "len(scratch2Write) = %d" % len(scratch2Write) return if darSteps[0] != scratch2Steps[0]: print "darSteps[0] = %d" % darSteps[0] print "scratch2Steps[0] = %d" % scratch2Steps[0] return if args.plot == "noplot": return steps = darSteps read = darRead - scratchRead - scratch2Read write = darWrite - scratchWrite - scratch2Write fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, steps, read, 'r', label='read', Ave=args.ave) Graph.timeSeries(ax, steps, write, 'b', label='write', Ave=args.ave) (handles, labels) = ax.get_legend_handles_labels() plt.xlabel('time') plt.ylabel(r'$MiB/sec$') plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') plt.title("%s %s Darshan minus LMT" % (day, fs)) if args.ybound is None: ax.set_ybound(lower = -50000, upper = 50000) else: ax.set_ybound(lower = -args.ybound, upper = args.ybound) if (not handles is None) and (not labels is None): plt.legend(handles, labels) if args.plot is None: plt.show() else: plt.savefig(args.plot) plt.cla() return
def do_plots(bulk, mode=None, plot=None, ybound=None, scale=1024.0*1024.0): for oss in bulk.OSSs: for ost in oss.OSTs: fig = plt.figure() ax = fig.add_subplot(111) steps = ost.Steps.Steps ymax = 0 if mode == 'Both': values = ost.OST.Values/scale max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='read and write', Ave=True) elif mode is None: values = ost.Read.Values/scale max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True) values = ost.Write.Values/scale max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True) elif mode == 'Read': values = ost.Read.Values/scale max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True) else: values = ost.Write.Values/scale max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True) plt.xlabel('time') plt.ylabel(r'$MiB/sec$') plt.legend() plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(ost.begin.sie)) plt.title("%s %s %s %s aggregate I/O" % (dayStr, bulk.name, oss.name, ost.name)) if ybound is None: ybound = ymax ax.set_ybound(lower = 0, upper = ybound) plt.savefig( plot + '/' + ost.name + '.png' ) plt.cla()
def doOSSCPUPlot(args, fsFile): if args.progress == True: print "OSS CPU plot" fsStepsGroup = fsFile['FSStepsGroup'] fsStepsDataSet = fsStepsGroup['FSStepsDataSet'] fs=fsStepsDataSet.attrs['fs'] try: host=fsStepsDataSet.attrs['host'] except: host='hopper' data = np.zeros(len(fsStepsDataSet)) ostReadGroup = fsFile['OSTReadGroup'] ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet'] ost_index = 0 for ost_name in ostBulkReadDataSet.attrs['OSTNames']: data += ostBulkReadDataSet[ost_index,:] ost_index += 1 ostWriteGroup = fsFile['OSTWriteGroup'] ostBulkWriteDataSet = ostWriteGroup['OSTBulkWriteDataSet'] ost_index = 0 for ost_name in ostBulkWriteDataSet.attrs['OSTNames']: data += ostBulkWriteDataSet[ost_index,:] ost_index += 1 cpu = np.zeros(len(fsStepsDataSet)) ossCPUGroup = fsFile['OSSCPUGroup'] ossCPUDataSet = ossCPUGroup['OSSCPUDataSet'] oss_index = 0 for oss_name in ossCPUDataSet.attrs['OSSNames']: cpu += ossCPUDataSet[oss_index,:] oss_index += 1 cpu /= oss_index fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, fsStepsDataSet, data, 'b', label='read plus write', Ave=True) plt.xlabel('time') plt.ylabel(r'$MiB/sec$') (handles, labels) = Graph.percent(ax, fsStepsDataSet, cpu, color='k', label='% CPU', Ave=True) plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') plt.title("%s %s aggregate I/O" % (fsStepsDataSet.attrs['day'], fsStepsDataSet.attrs['fs'])) ax.set_ybound(lower = 0, upper = 50000) if (not handles is None) and (not labels is None): plt.legend(handles, labels) path = os.path.dirname(args.file) if path == "": path = "." plt.savefig(path+'/'+host+'_'+fs+"_bulkBothCPU.png") plt.cla() return
def do_action(args, pair): """ I need to generate appropriate content here. """ (x, y) = pair fig = plt.figure() ax = fig.add_subplot(111) Graph.scatter(ax, x, y, 'b', label="just a dumb scatter plot") plt.xlabel('x') plt.ylabel(r'y') plt.title('x versus y') if args.plot == None: plt.show() else: plt.savefig(args.plot)
def do_xcorr(mds, plot=None, ybound=None, hilite="open"): if mds.CPU is None: print "mds.do_xcorr(): Error - There is no CPU utilization data for %s" % mds.name return if mds.MDS is None: print "mds.do_xcorr(): Error - There is no data for %s" % mds.name return if ybound is None: ybound = mds.MDS.getMax() fig = plt.figure() ax = fig.add_subplot(111) Graph.scatter(ax, mds.CPU.Values, mds.MDS.Values, 'b', label="mds ops") if not hilite is None: n = 0 op_index = None for op in mds.Ops: #print "mds.do_xcorr(): Debug - op name = %s, hilite = %s" % (op.name, hilite) if ((op.Values is None) or (op.Steps is None) or (op.Steps.steps() == 0) or (op.Stats is None)): continue if op.name == hilite: op_index = n n += 1 if n > 0: # mds.x is teh result of a generalized linear regression apporioning # the fraction of CPU utilization to each operation. x[n] is the # final category, of "no operation". if mds.x is None: mds.attribute() if (not op_index is None) and (mds.x[op_index] != 0.0): slope = (100 - mds.x[n])/(100*mds.x[op_index]) model_x = np.array(range(101)) model_y = mds.x[n]/100.0 + slope*model_x #print "hilite = %s, op_index = %d, y(0) = %f, y(1) = %f" % (hilite, op_index, model_y[0], model_y[99]) ax.plot(model_x, model_y, "r-", label=hilite) else: print "mds.do_xcorr(): Warning - No ops with data for regression" ax.set_xbound(lower = 0, upper = 100) ax.set_ybound(lower = 0, upper = ybound) ax.legend() plt.title("%s activity vs %%CPU" % mds.name) ax.set_xlabel(r"$\%$ CPU") ax.set_ylabel(r"$ops/sec$") if plot is None: plt.show() else: plt.savefig(plot) plt.cla()
def doAction(args, b_sie, e_sie, fsFile): fsStepsGroup = fsFile['FSStepsGroup'] fsStepsDataSet = fsStepsGroup['FSStepsDataSet'] if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]): print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]): print "The beginning timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return b_index = find_sie(b_sie, fsStepsDataSet) e_index = find_sie(e_sie, fsStepsDataSet) fs=fsStepsDataSet.attrs['fs'] try: host=fsStepsDataSet.attrs['host'] except: host='hopper' mds = np.zeros(e_index - b_index + 1) mdsOpsGroup = fsFile['MDSOpsGroup'] mdsOpsDataSet = mdsOpsGroup['MDSOpsDataSet'] op_index = 0 for op_name in mdsOpsDataSet.attrs['OpNames']: mds += mdsOpsDataSet[op_index,b_index:e_index+1] op_index += 1 mdsCPUGroup = fsFile['MDSCPUGroup'] mdsCPUDataSet = mdsCPUGroup['MDSCPUDataSet'] cpu = mdsCPUDataSet[b_index:e_index+1] np.set_printoptions(threshold='nan') #print "cpu: ", cpu fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], mds, 'g', label='metadata', Ave=True) plt.xlabel('time') plt.ylabel(r'$ops/sec$') (handles, labels) = Graph.percent(ax, fsStepsDataSet[b_index:e_index+1], cpu, color='k', label='% CPU', Ave=True) plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') plt.title("%s %s Metadata Operations" % (fsStepsDataSet.attrs['day'], fsStepsDataSet.attrs['fs'])) ax.set_ybound(lower = 0, upper = 120000) if (not handles is None) and (not labels is None): plt.legend(handles, labels) if args.plot is None: plt.show() else: plt.savefig(args.plot) plt.cla() return
def doMDSPlot(args, fsFile): if args.progress == True: print "MDS plot" fsStepsGroup = fsFile['FSStepsGroup'] fsStepsDataSet = fsStepsGroup['FSStepsDataSet'] intervals = np.zeros(len(fsStepsDataSet)) intervals[1:] = np.diff(fsStepsDataSet) fs=fsStepsDataSet.attrs['fs'] try: host=fsStepsDataSet.attrs['host'] except: host='hopper' mds = np.zeros(len(fsStepsDataSet)) mdsOpsGroup = fsFile['MDSOpsGroup'] mdsOpsDataSet = mdsOpsGroup['MDSOpsDataSet'] op_index = 0 for op_name in mdsOpsDataSet.attrs['OpNames']: mds += mdsOpsDataSet[op_index,:] op_index += 1 highVals, = np.where(mds > 100000) if len(highVals) > 0: print "Warning: Exceedingly high values reported for ", highVals print fsStepsDataSet[mds > 1000000] print mds[mds > 1000000] AggregateOps = np.sum(mds*intervals) cpu = np.zeros(len(fsStepsDataSet)) mdsCPUGroup = fsFile['MDSCPUGroup'] mdsCPUDataSet = mdsCPUGroup['MDSCPUDataSet'] fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, fsStepsDataSet, mds, 'b', label='mds', Ave=True) plt.xlabel('time') plt.ylabel(r'$Ops/sec$') (handles, labels) = Graph.percent(ax, fsStepsDataSet, mdsCPUDataSet, color='k', label='% CPU', Ave=True) plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') plt.title("%s %s Metadata Operations" % (fsStepsDataSet.attrs['day'], fsStepsDataSet.attrs['fs'])) ax.set_ybound(lower = 0, upper = 120000) if (not handles is None) and (not labels is None): plt.legend(handles, labels) path = os.path.dirname(args.file) if path == "": path = "." plt.savefig(path+'/'+host+'_'+fs+"_MDS+CPU.png") plt.cla() return(AggregateOps)
def time_plot_bin(args, BO): if not args.stat in BO.BrwNameDict: print "brwost.py.do_action(): %s not found in the set of Statistics" % args.stat return Bins = BO.Bins[BO.BrwNameDict[args.stat]] if not args.bin in Bins.BinDict: print "brwost.py.do_action(): %s not found in the set of Bins for %s" % (args.stat, Bins.name) return read = BO.Read[Bins.id] fig = plt.figure() ax = fig.add_subplot(111) steps = BO.Steps.Steps times = np.diff(steps) ymax = 0 values = read.Values[Bins.BinDict[args.bin],:] values[1:] = np.diff(values)/times values[0] = 0.0 max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, color='r', label='read', Ave=True, format='r-') write = BO.Write[Bins.id] values = write.Values[Bins.BinDict[args.bin],:] values[1:] = np.diff(values)/times values[0] = 0.0 max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, color='b', label='write', Ave=True, format='b-') plt.xlabel('time') plt.ylabel(r'$count/sec$') plt.legend() plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(BO.begin.sie)) plt.title("%s %s %s bin %d RPC stats" % (dayStr, BO.name, args.stat, args.bin)) if (not args.ybound is None) and (args.ybound > ymax): ymax = args.ybound ax.set_ybound(lower = 0, upper = ymax) if args.plot is None: plt.show() else: plt.savefig(args.plot) plt.cla() return
def doAction(args, b_sie, e_sie, fsFile): fsStepsGroup = fsFile['FSStepsGroup'] fsStepsDataSet = fsStepsGroup['FSStepsDataSet'] if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]): print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]): print "The beginning timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return b_index = find_sie(b_sie, fsStepsDataSet) e_index = find_sie(e_sie, fsStepsDataSet) #print b_index, e_index fs=fsStepsDataSet.attrs['fs'] try: host=fsStepsDataSet.attrs['host'] except: host='hopper' ossCPUGroup = fsFile["OSSCPUGroup"] ossCPUDataSet = ossCPUGroup["OSSCPUDataSet"] fsMissingGroup = fsFile["FSMissingGroup"] fsMissingDataSet = fsMissingGroup["FSMissingDataSet"] num_steps = e_index - b_index + 1 Missing = np.zeros(num_steps) oss_index = 0 for oss_name in ossCPUDataSet.attrs["OSSNames"]: Missing += fsMissingDataSet[oss_index,b_index:e_index+1] oss_index += 1 fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], Missing, 'r', label='missing', Ave=False, format='+') plt.xlabel('time') plt.ylabel(r'$count$') plt.legend() plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(b_sie)) plt.title("%s %s %s Missing Observations" % (dayStr, host, fs)) ax.set_ybound(lower = 0, upper = 30) if args.plot is None: plt.show() else: plt.savefig(args.plot) plt.cla()
def do_plot(name, steps, Plots, plot, ybound): if args.lines == True: format = '-' else: format = None fig = plt.figure() ax = fig.add_subplot(111) for Plot in Plots: Graph.timeSeries(ax, steps, Plot['values'], Plot['color'], Plot['label'], Ave=True, format=format) plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') plt.xlabel('time') plt.ylabel(r'$MiB/sec$') plt.legend() plt.title("%s I/O" % (name)) ax.set_ybound(lower=0, upper=ybound) if plot == None: plt.show() else: plt.savefig(plot) plt.cla()
def do_plot(bulk, plot=None, ybound=30): steps = bulk.Steps.Steps num_steps = len(steps) num_osss = bulk.getNumOSSs() Missing = np.zeros(num_steps) for oss in bulk.OSSs: Missing += oss.Missing fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, steps, Missing, 'r', label='missing', Ave=False, format='+') plt.xlabel('time') plt.ylabel(r'$count$') plt.legend() plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(bulk.begin.sie)) plt.title("%s %s Missing Observations" % (dayStr, bulk.name)) ax.set_ybound(lower = 0, upper = ybound) if plot is None: plt.show() else: plt.savefig(plot) plt.cla()
def doMissingPackets(args, fsFile): if args.progress == True: print "Missing packets plot" fsStepsGroup = fsFile['FSStepsGroup'] fsStepsDataSet = fsStepsGroup['FSStepsDataSet'] fs=fsStepsDataSet.attrs['fs'] try: host=fsStepsDataSet.attrs['host'] except: host='hopper' missing = np.zeros(len(fsStepsDataSet)) ossCPUGroup = fsFile["OSSCPUGroup"] ossCPUDataSet = ossCPUGroup["OSSCPUDataSet"] fsMissingGroup = fsFile['FSMissingGroup'] fsMissingDataSet = fsMissingGroup['FSMissingDataSet'] Missing = np.zeros(len(fsStepsDataSet)) oss_index = 0 for oss_name in ossCPUDataSet.attrs["OSSNames"]: Missing += fsMissingDataSet[oss_index,:] oss_index += 1 fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, fsStepsDataSet, Missing, 'r', label='misssing', Ave=False, format='+') plt.xlabel('time') plt.ylabel(r'$count$') plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') plt.title("%s %s Missing Packets" % (fsStepsDataSet.attrs['day'], fsStepsDataSet.attrs['fs'])) ax.set_ybound(lower = 0, upper = 156) plt.legend() path = os.path.dirname(args.file) if path == "": path = "." plt.savefig(path+'/'+host+'_'+fs+"_Missing.png") plt.cla() return
def do_xcorr(B, mode=None, plot=None, ybound=None, scale=1024*1024): """ The scatter plot of the aggregate I/O rates versus average CPU utilization is less illumination than the composite of all the individual OSS scatter plots. I don't call on this one, but it is still here and available. """ if not ((mode == 'Read') or (mode == 'Write') or (mode == 'Both') or (mode is None)): print "bulk.do_xcorr: Error - Unrecognized mode %s" % mode return if B.CPU is None: print "bulk.do_xcorr(): Error - There is no CPU utilization data for %s" % B.name return if ((B.Read is None) or (B.Write is None) or (B.bulk is None)): print "bulk.do_xcorr(): Error - There is no data" return(None) if ybound is None: if mode == 'Read': ymax = B.Read.getMax()/scale elif mode == 'Write': ymax = B.Write.getMax()/scale elif mode == 'Both': ymax = B.bulk.getMax()/scale else: readMax = B.Read.getMax()/scale writeMax = B.Write.getMax()/scale if readMax > writeMax: ymax = readMax else: ymax = writeMax ybound = ymax fig = plt.figure() ax = fig.add_subplot(111) if (mode is None) or (mode == 'Read'): Graph.scatter(ax, B.CPU.Values, B.Read.Values/scale, 'r', label="read") if (mode is None) or (mode == 'Write'): Graph.scatter(ax, B.CPU.Values, B.Write.Values/scale, 'b', label="write") if mode == 'Both': Graph.scatter(ax, B.CPU.Values, B.bulk.Values/scale, 'b', label="read+write") ax.set_xbound(lower = 0, upper = 100) ax.set_ybound(lower = 0, upper = ybound) ax.legend() dayStr = time.strftime("%Y-%m-%d", time.localtime(B.begin.sie)) plt.title("%s bulk %s activity vs %%CPU" % (dayStr, B.name)) ax.set_xlabel(r"$\%$ CPU") ax.set_ylabel(r"$MB/sec$") if plot is None: plt.show() else: plt.savefig(plot)
def do_composite_xcorr(B, mode=None, plot=None, ybound=None, scale=1024.0*1024.0): if ybound is None: ymax = 0.0 for oss in B.OSSs: if mode == 'Read': ossMax = oss.Read.getMax()/scale elif mode == 'Write': ossMax = oss.Write.getMax()/scale elif mode == 'Both': ossMax = oss.OSS.getMax()/scale else: readMax = oss.Read.getMax()/scale writeMax = oss.Write.getMax()/scale if readMax > writeMax: ossMax = readMax else: ossMax = writeMax if ossMax > ymax: ymax = ossMax ybound = ymax if ybound <= 0.0: print "bulk.do_composite_xcorr(): Warning - Failed to determine y-scale" return fig = plt.figure() ax = fig.add_subplot(111) handles = None labels = None for oss in B.OSSs: if (mode is None) or (mode == 'Read'): Graph.scatter(ax, oss.CPU.Values, oss.Read.Values/scale, 'r', label='read') if (mode is None) or (mode == 'Write'): Graph.scatter(ax, oss.CPU.Values, oss.Write.Values/scale, 'b', label='write') if mode == 'Both': Graph.scatter(ax, oss.CPU.Values, oss.OSS.Values/scale, 'b', label='read+write') if (handles is None) and (labels is None): (handles, labels) = ax.get_legend_handles_labels() plt.legend(handles, labels) ax.set_xbound(lower = 0, upper = 100) ax.set_ybound(lower = 0, upper = ybound) dayStr = time.strftime("%Y-%m-%d", time.localtime(B.begin.sie)) plt.title("%s Composite %s OSS activity vs %%CPU" % (dayStr, B.name)) ax.set_xlabel(r"$\%$ CPU") ax.set_ylabel(r"$MB/sec$") if plot is None: plt.show() else: plt.savefig(plot)
def do_xcorr(bulk, mode, plot, ymax=None, scale=1024*1024): """ """ if not ((mode == 'Read') or (mode == 'Write') or (mode == 'Both') or (mode == None)): print "bulk_quick.do_xcorr: Error - Unrecognized mode %s" % mode return if bulk.CPU == None: print "bulk_quick.do_xcorr(): Error - There is no CPU utilization data for %s" % bulk.name return if ((bulk.Read == None) or (bulk.Write == None) or (bulk.Bulk == None)): print "bulk_quick.do_xcorr(): Error - There is no data" return(None) if ymax == None: if mode == 'Read': ymax = bulk.Read.getMax()/scale elif mode == 'Write': ymax = bulk.Write.getMax()/scale elif mode == 'Both': ymax = bulk.Bulk.getMax()/scale else: readMax = bulk.Read.getMax()/scale writeMax = bulk.Write.getMax()/scale if readMax > writeMax: ymax = readMax else: ymax = writeMax fig = plt.figure() ax = fig.add_subplot(111) if (mode == None) or (mode == 'Read'): Graph.scatter(ax, bulk.CPU.Values, bulk.Read.Values/scale, 'r', label="read") if (mode == None) or (mode == 'Write'): Graph.scatter(ax, bulk.CPU.Values, bulk.Write.Values/scale, 'b', label="write") if mode == 'Both': Graph.scatter(ax, bulk.CPU.Values, bulk.Bulk.Values/scale, 'b', label="read+write") ax.set_xbound(lower = 0, upper = 100) ax.set_ybound(lower = 0, upper = ymax) ax.legend() dayStr = time.strftime("%Y-%m-%d", time.localtime(bulk.begin.sie)) plt.title("%s Bulk %s activity vs \%CPU" % (dayStr, bulk.name)) ax.set_xlabel(r"$\%$ CPU") ax.set_ylabel(r"$MB/sec$") if plot == None: plt.show() else: plt.savefig(plot)
def do_xcorr(oss, mode=None, plot=None, ybound=None, scale=1024.0*1024.0): if not ((mode == 'Read') or (mode == 'Write') or (mode == 'Both') or (mode is None)): print "oss.do_xcorr: Error - Unrecognized mode %s" % mode return if oss.CPU is None: print "oss.do_xcorr(): Error - There is no CPU utilization data for %s" % oss.name return if ((oss.Read is None) or (oss.Write is None) or (oss.OSS is None)): print "oss.do_xcorr(): Error - There is no data" return(None) if ybound is None: if mode == 'Read': ymax = oss.Read.getMax()/scale elif mode == 'Write': ymax = oss.Write.getMax()/scale elif mode == 'Both': ymax = oss.OSS.getMax()/scale else: readMax = oss.Read.getMax()/scale writeMax = oss.Write.getMax()/scale if readMax > writeMax: ymax = readMax else: ymax = writeMax ybound = ymax fig = plt.figure() ax = fig.add_subplot(111) if (mode is None) or (mode == 'Read'): Graph.scatter(ax, oss.CPU.Values, oss.Read.Values/scale, 'r', label="read") if (mode is None) or (mode == 'Write'): Graph.scatter(ax, oss.CPU.Values, oss.Write.Values/scale, 'b', label="write") if mode == 'Both': Graph.scatter(ax, oss.CPU.Values, oss.OSS.Values/scale, 'b', label="read+write") ax.set_xbound(lower = 0, upper = 100) ax.set_ybound(lower = 0, upper = ybound) ax.legend() dayStr = time.strftime("%Y-%m-%d", time.localtime(oss.begin.sie)) plt.title("%s OSS %s activity vs %%CPU" % (dayStr, oss.name)) ax.set_xlabel(r"$\%$ CPU") ax.set_ylabel(r"$MB/sec$") if plot is None: plt.show() else: plt.savefig(plot)
def do_action(args, bulk, brwfs): steps = bulk.Steps.Steps[1:] num_steps = len(steps) times = bulk.Steps.Diff # We don't want to distort the analysis by having some observation # intervals longer than the nominal five seconds, do divide by the # actual elapsed time to produce rates in MB/s, and Count/s if not args.ost is None: ost = bulk.getOST(ost=args.ost) if ost is None: print "we didn't find %s" % args.ost readMBpS = ost.Read.Values[1:] readMBpS /= (1024*1024) writeMBpS = ost.Write.Values[1:] writeMBpS /= (1024*1024) # no cpu caclulation for one OST cpu = np.zeros_like(readMBpS) brwost = brwfs.getOST(ost=ost.name) if brwost is None: print "we didn't find %s among the brw_stats" % ost.name return id = brwost.getStatId(args.stat) stat_bins = brwost.Bins[brwost.BrwIdDict[id]].Bins readHistpS = np.diff(brwost.Read[id].Values)/times readHistpS[np.where(readHistpS.mask==True)] = 0 writeHistpS = np.diff(brwost.Write[id].Values)/times writeHistpS[np.where(writeHistpS.mask==True)] = 0 else: readMBpS = bulk.Read.Values[1:] readMBpS /= (1024*1024) writeMBpS = bulk.Write.Values[1:] writeMBpS /= (1024*1024) cpu = bulk.CPU.Values[1:] readHistpS = None writeHistpS = None num_osts = 0 np.set_printoptions(threshold='nan') for oss in bulk.OSSs: for ost in oss.OSTs: print ost.name brwost = brwfs.getOST(ost=ost.name) id = brwost.getStatId(args.stat) stat_bins = brwost.Bins[brwost.BrwIdDict[id]].Bins readpS = np.diff(brwost.Read[id].Values)/times readpS[np.where(readpS.mask==True)] = 0 if readHistpS is None: readHistpS = np.zeros_like(readpS) print readpS readHistpS += readpS writepS = np.diff(brwost.Write[id].Values)/times writepS[np.where(writepS.mask==True)] = 0 if writeHistpS is None: writeHistpS = np.zeros_like(writepS) print writepS writeHistpS += writepS num_osts += 1 if (num_osts == 0) or (readHistpS is None) or (writeHistpS is None): print "we didn't get anything for the brw_stats data" return readHistpS /= num_osts writeHistpS /= num_osts histSeries = np.transpose(np.vstack((readHistpS, writeHistpS))) if args.report == True: print "%d steps" % len(steps) num_bins = len(stat_bins) # Now we want to construct the two element distillation of histSeries distill = np.zeros((2*num_bins, 4), dtype=np.float64) distill[0:num_bins,0] = 1.0 distill[0:num_bins,1] = stat_bins distill[num_bins:2*num_bins,2] = 1.0 distill[num_bins:2*num_bins,3] = stat_bins A = np.matrix(histSeries)*np.matrix(distill) # This is the result of the two_element_model.py calculation: x = np.matrix([1.62589065e-03, 4.39766334e-09, 3.23092722e-03, 1.72900072e-09]) yhat = A * np.transpose(x) if args.report == True: print "yhat:" print yhat factor = 1 if args.hist == True: if np.max(yhat) > 1.0: factor = int(np.max(yhat)) + 1 utilHist, utilBins = np.histogram(yhat*100.0, bins=100, range=(0.0, 100.0*factor)) if args.report == True: print "utilBins:" print utilBins print "utilHist:" print utilHist if args.plot == "noplot": return fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, steps, readMBpS, 'r', label='read', Ave=False) Graph.timeSeries(ax, steps, writeMBpS, 'b', label='write', Ave=False) plt.xlabel('time') plt.ylabel(r'$MiB/sec$') (handles, labels) = Graph.percent(ax, steps, yhat*100.0, 'k', format="-", label='pct FSU', Ave=False) if (not handles is None) and (not labels is None): plt.legend(handles, labels) plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(bulk.begin.sie)) plt.title("%s %s File System Utilization" % (dayStr, bulk.name)) if args.ybound is None: ax.set_ybound(lower=0) else: ax.set_ybound(lower=0, upper=args.ybound) if args.plot is None: plt.show() else: plt.savefig(args.plot) plt.cla() if args.hist == False: return fig = plt.figure() ax = fig.add_subplot(111) ax.bar(utilBins[1:], utilHist, color='k') plt.xlabel('pct fs util') plt.ylabel('count') plt.title('Distribution of File System Utilization') if args.plot is None: plt.show() else: plt.savefig('hist_'+args.plot) plt.cla() return
def do_plot(oss, mode=None, plot=None, ybound=None, scale=1024.0*1024.0): if args.lines == True: format = '-' else: format = None fig = plt.figure() ax = fig.add_subplot(111) steps = oss.Steps.Steps ymax = 0 if mode == 'Both': values = oss.OSS.Values/scale max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='read and write', Ave=True, format=format) elif mode is None: values = oss.Read.Values/scale max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True, format=format) values = oss.Write.Values/scale max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True, format=format) elif mode == 'Read': values = oss.Read.Values/scale max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True, format=format) else: values = oss.Write.Values/scale max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True) plt.xlabel('time') plt.ylabel(r'$MiB/sec$') if not oss.CPU is None: values = oss.CPU.Values (handles, labels) = Graph.percent(ax, steps, values, 'k', label='% CPU', Ave=True) if (not handles is None) and (not labels is None): plt.legend(handles, labels) else: print "oss.do_plot(): Warning - Plotting CPU utilization failed." else: plt.legend() plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(oss.begin.sie)) plt.title("%s %s aggregate I/O" % (dayStr, oss.name)) if ybound is None: ybound = ymax ax.set_ybound(lower = 0, upper = ybound) if plot is None: plt.show() else: plt.savefig(plot) plt.cla()
def doAction(args, b_sie, e_sie, fsFile): fsStepsGroup = fsFile['FSStepsGroup'] fsStepsDataSet = fsStepsGroup['FSStepsDataSet'] if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]): print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]): print "The ending timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return b_index = find_sie(b_sie, fsStepsDataSet) e_index = find_sie(e_sie, fsStepsDataSet) #print "data from index %d to %d" % (b_index, e_index) fs=fsStepsDataSet.attrs['fs'] try: host=fsStepsDataSet.attrs['host'] except: host='hopper' read = np.zeros(e_index - b_index + 1) ostReadGroup = fsFile['OSTReadGroup'] ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet'] ost_index = 0 if args.report == True: np.set_printoptions(threshold='nan') for ost_name in ostBulkReadDataSet.attrs['OSTNames']: #print "OST %s" % ost_name read += ostBulkReadDataSet[ost_index,b_index:e_index+1] #print "read:", ostBulkReadDataSet[ost_index,b_index:e_index+1] ost_index += 1 read /= (1024*1024) write = np.zeros(e_index - b_index + 1) ostWriteGroup = fsFile['OSTWriteGroup'] ostBulkWriteDataSet = ostWriteGroup['OSTBulkWriteDataSet'] ost_index = 0 for ost_name in ostBulkWriteDataSet.attrs['OSTNames']: #print "OST %s" % ost_name write += ostBulkWriteDataSet[ost_index,b_index:e_index+1] #print "write: ", ostBulkWriteDataSet[ost_index,b_index:e_index+1] ost_index += 1 write /= (1024*1024) cpu = np.zeros(e_index - b_index + 1) ossCPUGroup = fsFile['OSSCPUGroup'] ossCPUDataSet = ossCPUGroup['OSSCPUDataSet'] oss_index = 0 for oss_name in ossCPUDataSet.attrs['OSSNames']: cpu += ossCPUDataSet[oss_index,b_index:e_index+1] oss_index += 1 cpu /= oss_index if args.report == True: print "read:", read print "write: ", write print "cpu: ", cpu if args.plot == "noplot": return fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], read, 'r', label='read', Ave=args.ave) Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], write, 'b', label='write', Ave=args.ave) (handles, labels) = ax.get_legend_handles_labels() plt.xlabel('time') plt.ylabel(r'$MiB/sec$') if args.cpu == True: (handles, labels) = Graph.percent(ax, fsStepsDataSet[b_index:e_index+1], cpu, color='k', label='% CPU', Ave=args.ave) plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') plt.title("%s %s aggregate I/O" % (fsStepsDataSet.attrs['day'], fsStepsDataSet.attrs['fs'])) if args.ybound is None: ax.set_ybound(lower = 0, upper = 50000) else: ax.set_ybound(lower = 0, upper = args.ybound) if (not handles is None) and (not labels is None): plt.legend(handles, labels) if args.plot is None: plt.show() else: plt.savefig(args.plot) plt.cla() return
def doAction(args, b_sie, e_sie, darFile): StepsGroup = darFile['DarStepsGroup'] StepsDataSet = StepsGroup['DarStepsDataSet'] if (b_sie < StepsDataSet[0]) or (b_sie > StepsDataSet[-1]): print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, StepsDataSet[0], StepsDataSet[-1]) return if (e_sie < StepsDataSet[0]) or (e_sie > StepsDataSet[-1]): print "The ending timestamp %d is outside the date range from %d to %d" % (e_sie, StepsDataSet[0], StepsDataSet[-1]) return b_index = find_sie(b_sie, StepsDataSet) e_index = find_sie(e_sie, StepsDataSet) #print "data from index %d to %d" % (b_index, e_index) fs=StepsDataSet.attrs['fs'] try: host=StepsDataSet.attrs['host'] except: host='hopper' read = np.zeros(e_index - b_index + 1) DarshanGroup = darFile['DarshanGroup'] readIODataSet = DarshanGroup['ReadIODataSet'] if args.report == True: np.set_printoptions(threshold='nan') read = readIODataSet[b_index:e_index+1] #print "read:", readIODataSet[b_index:e_index+1] read /= (1024*1024) maxRead = np.max(read) write = np.zeros(e_index - b_index + 1) writeIODataSet = DarshanGroup['WriteIODataSet'] write = writeIODataSet[b_index:e_index+1] #print "write: ", writeIODataSet[b_index:e_index+1] write /= (1024*1024) maxWrite = np.max(write) if args.verbose == True: print "read:", read print "write: ", write if args.report == True: print "ave read: %f" % np.average(read) print "ave write:%f" % np.average(write) print "max read: %f" % maxRead print "max write:%f" % maxWrite if args.plot == "noplot": return max = maxWrite if maxRead > maxWrite: max = maxRead scale = makeRound(max) fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, StepsDataSet[b_index:e_index+1], read, 'r', label='read', Ave=args.ave) Graph.timeSeries(ax, StepsDataSet[b_index:e_index+1], write, 'b', label='write', Ave=args.ave) (handles, labels) = ax.get_legend_handles_labels() plt.xlabel('time') plt.ylabel(r'$MiB/sec$') plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') plt.title("%s %s aggregate I/O" % (StepsDataSet.attrs['day'], StepsDataSet.attrs['fs'])) if args.ybound is None: ax.set_ybound(lower = 0, upper = scale) else: ax.set_ybound(lower = 0, upper = args.ybound) if (not handles is None) and (not labels is None): plt.legend(handles, labels) if args.plot is None: plt.show() else: plt.savefig(args.plot) plt.cla() return
def do_action(args, b_sie, e_sie, fsFile): fsStepsGroup = fsFile['FSStepsGroup'] fsStepsDataSet = fsStepsGroup['FSStepsDataSet'] if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]): print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]): print "The ending timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return b_index = find_sie(b_sie, fsStepsDataSet) e_index = find_sie(e_sie, fsStepsDataSet) if b_index == None: print "brw_stats_model_h5lmt.do_action(): Failed to find timestamp index for %d" % b_sie return if e_index == None: print "brw_stats_model_h5lmt.do_action(): Failed to find timestamp index for %d" % e_sie return stepsDiff = np.zeros_like(fsStepsDataSet[b_index:e_index+1], dtype=np.int32) stepsDiff[1:] = np.diff(fsStepsDataSet[b_index:e_index+1]) fs=fsStepsDataSet.attrs['fs'] try: host=fsStepsDataSet.attrs['host'] except: host='hopper' ostReadGroup = fsFile['OSTReadGroup'] ostWriteGroup = fsFile['OSTWriteGroup'] ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet'] ostIosizeReadDataSet = ostReadGroup['OSTIosizeReadDataSet'] if args.stat != ostIosizeReadDataSet.attrs['stat']: print "We should only be seeing BRW_IOSIZE statistics not %s" % ostIosizeReadDataSet.attrs['stat'] return bins = np.matrix(ostIosizeReadDataSet.attrs['bins']) ostIosizeWriteDataSet = ostWriteGroup['OSTIosizeWriteDataSet'] ostHist = None ContribDS = None ost_index = 0 if args.report == True: np.set_printoptions(threshold='nan') for ost_name in ostBulkReadDataSet.attrs['OSTNames']: if args.progress == True: print "OST %d: %s" % (ost_index, ost_name) if args.ost != ost_name: ost_index += 1 continue readpS = np.transpose(np.matrix(ostIosizeReadDataSet[ost_index,:,b_index:e_index+1])) #print readpS writepS = np.transpose(np.matrix(ostIosizeWriteDataSet[ost_index,:,b_index:e_index+1])) #print writepS if args.verbose == True: print "%d steps" % len(fsStepsDataSet) num_bins = len(bins) # Now we want to construct the two element distillation of histSeries read = np.array(readpS*np.transpose(bins)).flatten() read /= 1024.0*1024.0 ones = np.transpose(np.matrix(np.ones_like(bins, dtype=np.float64))) readIOs = np.array(readpS*ones).flatten() write = np.array(writepS*np.transpose(bins)).flatten() write /= 1024.0*1024.0 writeIOs = np.array(writepS*ones).flatten() theOst = ost_name break if ost_index == 0: print "we didn't get anything for the brw_stats data" return if args.plot == "noplot": return fig = plt.figure() ax = fig.add_subplot(111) if args.iops == True: Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], readIOs, 'r', label='read', Ave=args.ave) Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], writeIOs, 'b', label='write', Ave=args.ave) plt.ylabel('IOPS') plt.title('IOPS on OST %s' % theOst) else: Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], read, 'r', label='read', Ave=args.ave) Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], write, 'b', label='write', Ave=args.ave) plt.ylabel('MB/s') plt.title('data rate on OST %s' % theOst) plt.xlabel('time') plt.legend() if not args.ybound is None: ax.set_ybound(lower = 0, upper = args.ybound) if args.plot is None: plt.show() else: plt.savefig(args.plot) plt.cla() return
def doAction(args, b_sie, e_sie, fsFile): fsStepsGroup = fsFile['FSStepsGroup'] fsStepsDataSet = fsStepsGroup['FSStepsDataSet'] if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]): print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]): print "The ending timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return b_index = find_sie(b_sie, fsStepsDataSet) e_index = find_sie(e_sie, fsStepsDataSet) #print "data from index %d to %d" % (b_index, e_index) fs=fsStepsDataSet.attrs['fs'] # I do this for backward compatability. A few early h5lmt file did not define the host try: host=fsStepsDataSet.attrs['host'] except: host='hopper' read = np.zeros(e_index - b_index + 1) ostReadGroup = fsFile['OSTReadGroup'] ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet'] write = np.zeros(e_index - b_index + 1) ostWriteGroup = fsFile['OSTWriteGroup'] ostBulkWriteDataSet = ostWriteGroup['OSTBulkWriteDataSet'] ostIosizeReadDataSet = ostReadGroup['OSTIosizeReadDataSet'] bins = ostIosizeReadDataSet.attrs['bins'] ostIosizeWriteDataSet = ostWriteGroup['OSTIosizeWriteDataSet'] ost_index = 0 if args.report == True: np.set_printoptions(threshold='nan') for ost_name in ostBulkReadDataSet.attrs['OSTNames']: read += ostBulkReadDataSet[ost_index,b_index:e_index+1] write += ostBulkWriteDataSet[ost_index,b_index:e_index+1] readpS = ostIosizeReadDataSet[ost_index,:,b_index:e_index+1] writepS = ostIosizeWriteDataSet[ost_index,:,b_index:e_index+1] readIOBytes = np.array(np.matrix(bins)*np.matrix(readpS)) writeIOBytes = np.array(np.matrix(bins)*np.matrix(writepS)) read -= readIOBytes[0] write -= writeIOBytes[0] ost_index += 1 if ost_index == 0: print "we didn't get any data" return read /= (1024*1024) write /= (1024*1024) if args.report == True: np.set_printoptions(threshold='nan') print "read:", read print "write: ", write if args.plot == "noplot": return fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], read, 'r', label='read', Ave=False) Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], write, 'b', label='write', Ave=False) (handles, labels) = ax.get_legend_handles_labels() plt.xlabel('time') plt.ylabel(r'$MiB/sec$') plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') plt.title("%s %s difference between bytes and I/Os" % (fsStepsDataSet.attrs['day'], fsStepsDataSet.attrs['fs'])) if args.ybound is None: ax.set_ybound(lower = -1000, upper = 50000) else: ax.set_ybound(lower = -1000, upper = args.ybound) if (not handles is None) and (not labels is None): plt.legend(handles, labels) if args.plot is None: plt.show() else: plt.savefig(args.plot) plt.cla() return
def do_plot(bulk, mode, plot, scale): fig = plt.figure() ax = fig.add_subplot(111) steps = bulk.Steps.Steps ymax = 0 if mode == 'Both': values = bulk.Bulk.Values/(1024*1024) max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='read and write', Ave=True) elif mode == None: values = bulk.Read.Values/(1024*1024) max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True) values = bulk.Write.Values/(1024*1024) max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True) elif mode == 'Read': values = bulk.Read.Values/(1024*1024) max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True) else: values = bulk.Write.Values/(1024*1024) max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True) plt.xlabel('time') plt.ylabel(r'$MiB/sec$') if bulk.CPU != None: values = bulk.CPU.Values (handles, labels) = Graph.percent(ax, steps, values, 'k', label='% CPU', Ave=True) if (handles != None) and (labels != None): plt.legend(handles, labels) else: print "bulk_quick.do_plot(): Warning - Plotting CPU utilization failed." else: plt.legend() plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(bulk.begin.sie)) plt.title("%s %s aggregate I/O" % (dayStr, bulk.name)) if scale == None: scale = ymax ax.set_ybound(lower = 0, upper = scale) if plot == None: plt.show() else: plt.savefig(plot) plt.cla()
def do_plot(B, mode=None, plot=None, ybound=None, scale=1024.0*1024.0, withCPU=True): if args.lines == True: format = '-' else: format = None fig = plt.figure() ax = fig.add_subplot(111) steps = B.Steps.Steps ymax = 0 np.set_printoptions(threshold='nan') if mode == 'Both': values = B.Bulk.Values/scale max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='read and write', Ave=True, format=format) elif mode is None: values = B.Read.Values/scale #print "read: ", values max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True, format=format) values = B.Write.Values/scale #print "write: ", values max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True, format=format) elif mode == 'Read': values = B.Read.Values/scale #print "read: ", values max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True, format=format) else: values = B.Write.Values/scale #print "write: ", values max = np.max(values) if max > ymax: ymax = max Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True, format=format) plt.xlabel('time') plt.ylabel(r'$MiB/sec$') if (withCPU == True) and (not B.CPU is None): values = np.array(B.CPU.Values) #values[np.where(values.mask==True)] = 0.0 (handles, labels) = Graph.percent(ax, steps, values, 'k', label='pct CPU', Ave=True) # insert bogus here for testing if (not handles is None) and (not labels is None): plt.legend(handles, labels) else: print "bulk.do_plot(): Warning - Plotting CPU utilization failed." else: plt.legend() plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(B.begin.sie)) plt.title("%s %s aggregate I/O" % (dayStr, B.name)) if ybound is None: ybound = ymax ax.set_ybound(lower = 0, upper = ybound) if plot is None: plt.show() else: plt.savefig(plot) plt.cla()
def doAction(args, b_sie, e_sie, fsFile): fsStepsGroup = fsFile['FSStepsGroup'] fsStepsDataSet = fsStepsGroup['FSStepsDataSet'] if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]): print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]): print "The ending timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return b_index = find_sie(b_sie, fsStepsDataSet) e_index = find_sie(e_sie, fsStepsDataSet) if args.report == True: print "data from index %d to %d" % (b_index, e_index) fs=fsStepsDataSet.attrs['fs'] try: host=fsStepsDataSet.attrs['host'] except: host='hopper' ostReadGroup = fsFile['OSTReadGroup'] ostWriteGroup = fsFile['OSTWriteGroup'] ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet'] ostIosizeReadDataSet = ostReadGroup['OSTIosizeReadDataSet'] if args.stat != ostIosizeReadDataSet.attrs['stat']: print "We should only be seeing BRW_IOSIZE statistics not %s" % ostIosizeReadDataSet.attrs['stat'] return bins = ostIosizeReadDataSet.attrs['bins'] ostIosizeWriteDataSet = ostWriteGroup['OSTIosizeWriteDataSet'] readHistpS = None writeHistpS = None ost_index = 0 foundOST = False for ost_name in ostBulkReadDataSet.attrs['OSTNames']: if args.progress == True: print "OST %d: %s" % (ost_index, ost_name) if not args.ost is None: if args.ost == ost_name: foundOST = True else: ost_index += 1 continue readpS = ostIosizeReadDataSet[ost_index,:,b_index:e_index+1] if readHistpS is None: readHistpS = np.zeros_like(readpS) #print readpS readHistpS += readpS writepS = ostIosizeWriteDataSet[ost_index,:,b_index:e_index+1] if writeHistpS is None: writeHistpS = np.zeros_like(writepS) #print writepS writeHistpS += writepS ost_index += 1 if (not args.ost is None) and (foundOST == False): print "%s not found" % args.ost return if (ost_index == 0) or (readHistpS is None) or (writeHistpS is None): print "we didn't get anything for the brw_stats data" return readIosizeHist = np.sum(readHistpS, axis=1) writeIosizeHist = np.sum(writeHistpS, axis=1) if not args.save is None: fsFile = h5py.File(args.save, 'a') try: fsIosizeGroup = fsFile["FSIosizeGroup"] except KeyError: fsIosizeGroup = fsFile.create_group("FSIosizeGroup") try: fsIosizeBinsDataSet = fsIosizeGroup["FSIosizeBinsDataSet"] fsIosizeBinsDataSet = bins except KeyError: fsIosizeBinsDataSet = fsIosizeGroup.create_dataset("FSIosizeBinsDataSet", data=bins) try: fsIosizeReadDataSet = fsIosizeGroup["FSIosizeReadDataSet"] fsIosizeReadDataSet = readIosizeHist except KeyError: fsIosizeReadDataSet = fsIosizeGroup.create_dataset("FSIosizeReadDataSet", data=readIosizeHist) try: fsIosizeWriteDataSet = fsIosizeGroup["FSIosizeWriteDataSet"] fsIosizeWriteDataSet = writeIosizeHist except: fsIosizeWriteDataSet = fsIosizeGroup.create_dataset("FSIosizeWriteDataSet", data=writeIosizeHist) fsIosizeGroup.attrs['day'] = fsStepsDataSet.attrs['day'] fsIosizeGroup.attrs['nextday'] = fsStepsDataSet.attrs['nextday'] fsIosizeGroup.attrs['host'] = fsStepsDataSet.attrs['host'] fsIosizeGroup.attrs['fs'] = fsStepsDataSet.attrs['fs'] fsIosizeGroup.attrs['stat'] = ostIosizeReadDataSet.attrs['stat'] fsIosizeGroup.attrs['bins'] = ostIosizeReadDataSet.attrs['bins'] fsFile.close() if args.plot == "noplot": return fig = plt.figure() fig.subplots_adjust(bottom=0.2) ax = fig.add_subplot(111) width = 0.35 x = np.arange(len(bins)) offset=0.0 Graph.bar(ax, x, readIosizeHist, width=width, offset=offset, color='r', label='read') offset += width Graph.bar(ax, x, writeIosizeHist, width=width, offset=offset, color='b', label='write') offset += width ax.set_ylabel('Count') ax.set_xlabel('MB') ax.set_title('%s %s I/O size histogram' % (host, fs) ) ax.set_xticks(x+width) ax.set_xticklabels( bins, rotation=45, horizontalalignment='right' ) ax.legend() if args.plot is None: plt.show() else: plt.savefig( args.plot ) plt.cla() return
def do_action(args, b_sie, e_sie, fsFile): fsStepsGroup = fsFile['FSStepsGroup'] fsStepsDataSet = fsStepsGroup['FSStepsDataSet'] if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]): print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]): print "The ending timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return b_index = find_sie(b_sie, fsStepsDataSet) e_index = find_sie(e_sie, fsStepsDataSet) if b_index == None: print "brw_stats_model_h5lmt.do_action(): Failed to find timestamp index for %d" % b_sie return if e_index == None: print "brw_stats_model_h5lmt.do_action(): Failed to find timestamp index for %d" % e_sie return fs=fsStepsDataSet.attrs['fs'] try: host=fsStepsDataSet.attrs['host'] except: host='hopper' readMBpS = np.zeros(e_index - b_index + 1) ostReadGroup = fsFile['OSTReadGroup'] ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet'] ost_index = 0 for ost_name in ostBulkReadDataSet.attrs['OSTNames']: readMBpS += ostBulkReadDataSet[ost_index,b_index:e_index+1] ost_index += 1 readMBpS /= 1024*1024 np.set_printoptions(threshold='nan') writeMBpS = np.zeros(e_index - b_index + 1) ostWriteGroup = fsFile['OSTWriteGroup'] ostBulkWriteDataSet = ostWriteGroup['OSTBulkWriteDataSet'] ost_index = 0 for ost_name in ostBulkWriteDataSet.attrs['OSTNames']: writeMBpS += ostBulkWriteDataSet[ost_index,b_index:e_index+1] ost_index += 1 writeMBpS /= 1024*1024 ostIosizeReadDataSet = ostReadGroup['OSTIosizeReadDataSet'] if args.stat != ostIosizeReadDataSet.attrs['stat']: print "We should only be seeing BRW_IOSIZE statistics not %s" % ostIosizeReadDataSet.attrs['stat'] return bins = ostIosizeReadDataSet.attrs['bins'] ostIosizeWriteDataSet = ostWriteGroup['OSTIosizeWriteDataSet'] readHistpS = None writeHistpS = None ost_index = 0 if args.report == True: np.set_printoptions(threshold='nan') for ost_name in ostBulkReadDataSet.attrs['OSTNames']: if args.progress == True: print "OST %d: %s" % (ost_index, ost_name) readpS = ostIosizeReadDataSet[ost_index,:,b_index:e_index+1] if readHistpS is None: readHistpS = np.zeros_like(readpS) #print readpS readHistpS += readpS writepS = ostIosizeWriteDataSet[ost_index,:,b_index:e_index+1] if writeHistpS is None: writeHistpS = np.zeros_like(writepS) #print writepS writeHistpS += writepS ost_index += 1 if (ost_index == 0) or (readHistpS is None) or (writeHistpS is None): print "we didn't get anything for the brw_stats data" return readHistpS /= ost_index writeHistpS /= ost_index histSeries = np.transpose(np.vstack((readHistpS, writeHistpS))) if args.report == True: print "%d steps" % len(fsStepsDataSet) num_bins = len(bins) # Now we want to construct the two element distillation of histSeries distill = np.zeros((2*num_bins, 4), dtype=np.float64) distill[0:num_bins,0] = 1.0 distill[0:num_bins,1] = bins distill[num_bins:2*num_bins,2] = 1.0 distill[num_bins:2*num_bins,3] = bins A = np.matrix(histSeries)*np.matrix(distill) # This is the result of the two_element_model.py calculation: x = np.matrix([1.62589065e-03, 4.39766334e-09, 3.23092722e-03, 1.72900072e-09]) yhat = A * np.transpose(x) yhat = np.array(yhat) yhat *= 100.0 if args.report == True: print "yhat (percent):" print yhat if args.hist == True: doHist(args, histSeries, yhat, fsStepsDataSet) if args.plot == "noplot": return fig = plt.figure() ax = fig.add_subplot(111) Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], readMBpS, 'r', label='read', Ave=False) Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], writeMBpS, 'b', label='write', Ave=False) plt.xlabel('time') plt.ylabel(r'$MiB/sec$') (handles, labels) = Graph.percent(ax, fsStepsDataSet[b_index:e_index+1], yhat, 'k', label='FSU', Ave=False, ybound=200.0) if (not handles is None) and (not labels is None): plt.legend(handles, labels) plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right') dayStr = time.strftime("%Y-%m-%d", time.localtime(fsStepsDataSet[0])) plt.title("%s %s File System Utilization" % (dayStr, fs)) if args.ybound is None: ax.set_ybound(lower=0, upper=50000) else: ax.set_ybound(lower=0, upper=args.ybound) if args.plot is None: plt.show() else: plt.savefig(args.plot) return