def do_plot(mds, plot, ymax, extra=None):
fig = plt.figure()
ax = fig.add_subplot(111)
steps = mds.Steps.Steps
values = mds.MDS.Values
if ymax is None:
ymax = np.max(values)
Graph.timeSeries(ax, steps, values, 'b', label='MDS', Ave=True)
plt.xlabel('time')
plt.ylabel(r'$ops/sec$')
if not mds.CPU is None:
values = mds.CPU.Values
(handles, labels) = Graph.percent(ax, steps, values, 'k',
label='% CPU', Ave=True)
if (not handles is None) and (not labels is None):
plt.legend(handles, labels)
else:
print "mds.do_plot(): Warning - Plotting CPU utilization failed."
else:
plt.legend()
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
start_time = steps[0]/(24.0*60.0*60.0) + mpl.dates.date2num(datetime.date(1970,1,1))
plt.title("%s metadata operations for %s" %
(mds.name,
mpl.dates.num2date(start_time).strftime("%Y-%m-%d"))
)
if ymax is None:
ymax = ymax
ax.set_ybound(lower=0, upper=ymax)
if plot is None:
plt.show()
else:
plt.savefig(plot)
plt.cla()
def show_metadata(args, metadata):
"""
It would be nice if I could unify this with Graph.mds_plot().
"""
fig = plt.figure()
ax = fig.add_subplot(111)
steps = metadata.Steps.Steps
values = metadata.MDS.Values
Graph.timeSeries(ax, steps, values, 'b', label='ops', Ave=True)
plt.xlabel('time')
plt.ylabel(r'$ops/sec$')
if not metadata.CPU is None:
values = metadata.CPU.Values
(handles, labels) = Graph.percent(ax, steps, values, 'k', label='% CPU', Ave=True)
plt.legend(handles, labels)
else:
plt.legend()
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
dayStr = time.strftime("%Y-%m-%d", time.localtime(metadata.begin.sie))
plt.title("%s %s metadata" % (dayStr, metadata.name))
page = StringIO.StringIO()
plt.savefig(page, format='png')
page.seek(0)
plt.cla()
return(page)
def show_bulk(args, bulk):
"""
It would be nice if I could unify this with Graph.bulk_plot().That
would entail using the StringIO function there and handling the
save/show option in the caller.
"""
scale = 1024.0*1024.0
fig = plt.figure()
ax = fig.add_subplot(111)
steps = bulk.Steps.Steps
ybound = 50000.0
values = bulk.Read.Values/scale
Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True)
values = bulk.Write.Values/scale
Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True)
plt.xlabel('time')
plt.ylabel(r'$MiB/sec$')
if not bulk.CPU is None:
values = bulk.CPU.Values
(handles, labels) = Graph.percent(ax, steps, values, 'k', label='% CPU', Ave=True)
plt.legend(handles, labels)
else:
plt.legend()
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
dayStr = time.strftime("%Y-%m-%d", time.localtime(bulk.begin.sie))
plt.title("%s %s aggregate I/O" % (dayStr, bulk.name))
ax.set_ybound(lower = 0, upper = ybound)
page = StringIO.StringIO()
plt.savefig(page, format='png')
page.seek(0)
plt.cla()
return(page)
def do_plot(bulk, mode, plot, scale):
fig = plt.figure()
ax = fig.add_subplot(111)
steps = bulk.Steps.Steps
ymax = 0
if mode == 'Both':
values = bulk.Bulk.Values/(1024*1024)
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'b', label='read and write', Ave=True)
elif mode == None:
values = bulk.Read.Values/(1024*1024)
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True)
values = bulk.Write.Values/(1024*1024)
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True)
elif mode == 'Read':
values = bulk.Read.Values/(1024*1024)
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'r', label='read', Ave=True)
else:
values = bulk.Write.Values/(1024*1024)
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'b', label='write', Ave=True)
plt.xlabel('time')
plt.ylabel(r'$MiB/sec$')
if bulk.CPU != None:
values = bulk.CPU.Values
(handles, labels) = Graph.percent(ax, steps, values, 'k', label='% CPU', Ave=True)
if (handles != None) and (labels != None):
plt.legend(handles, labels)
else:
print "bulk_quick.do_plot(): Warning - Plotting CPU utilization failed."
else:
plt.legend()
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
dayStr = time.strftime("%Y-%m-%d", time.localtime(bulk.begin.sie))
plt.title("%s %s aggregate I/O" % (dayStr, bulk.name))
if scale == None:
scale = ymax
ax.set_ybound(lower = 0, upper = scale)
if plot == None:
plt.show()
else:
plt.savefig(plot)
plt.cla()
def doOSSCPUPlot(args, fsFile):
if args.progress == True:
print "OSS CPU plot"
fsStepsGroup = fsFile['FSStepsGroup']
fsStepsDataSet = fsStepsGroup['FSStepsDataSet']
fs=fsStepsDataSet.attrs['fs']
try:
host=fsStepsDataSet.attrs['host']
except:
host='hopper'
data = np.zeros(len(fsStepsDataSet))
ostReadGroup = fsFile['OSTReadGroup']
ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet']
ost_index = 0
for ost_name in ostBulkReadDataSet.attrs['OSTNames']:
data += ostBulkReadDataSet[ost_index,:]
ost_index += 1
ostWriteGroup = fsFile['OSTWriteGroup']
ostBulkWriteDataSet = ostWriteGroup['OSTBulkWriteDataSet']
ost_index = 0
for ost_name in ostBulkWriteDataSet.attrs['OSTNames']:
data += ostBulkWriteDataSet[ost_index,:]
ost_index += 1
cpu = np.zeros(len(fsStepsDataSet))
ossCPUGroup = fsFile['OSSCPUGroup']
ossCPUDataSet = ossCPUGroup['OSSCPUDataSet']
oss_index = 0
for oss_name in ossCPUDataSet.attrs['OSSNames']:
cpu += ossCPUDataSet[oss_index,:]
oss_index += 1
cpu /= oss_index
fig = plt.figure()
ax = fig.add_subplot(111)
Graph.timeSeries(ax, fsStepsDataSet, data, 'b', label='read plus write', Ave=True)
plt.xlabel('time')
plt.ylabel(r'$MiB/sec$')
(handles, labels) = Graph.percent(ax, fsStepsDataSet, cpu, color='k', label='% CPU', Ave=True)
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
plt.title("%s %s aggregate I/O" % (fsStepsDataSet.attrs['day'],
fsStepsDataSet.attrs['fs']))
ax.set_ybound(lower = 0, upper = 50000)
if (not handles is None) and (not labels is None):
plt.legend(handles, labels)
path = os.path.dirname(args.file)
if path == "":
path = "."
plt.savefig(path+'/'+host+'_'+fs+"_bulkBothCPU.png")
plt.cla()
return
def doAction(args, b_sie, e_sie, fsFile):
fsStepsGroup = fsFile['FSStepsGroup']
fsStepsDataSet = fsStepsGroup['FSStepsDataSet']
if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]):
print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1])
return
if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]):
print "The beginning timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1])
return
b_index = find_sie(b_sie, fsStepsDataSet)
e_index = find_sie(e_sie, fsStepsDataSet)
fs=fsStepsDataSet.attrs['fs']
try:
host=fsStepsDataSet.attrs['host']
except:
host='hopper'
mds = np.zeros(e_index - b_index + 1)
mdsOpsGroup = fsFile['MDSOpsGroup']
mdsOpsDataSet = mdsOpsGroup['MDSOpsDataSet']
op_index = 0
for op_name in mdsOpsDataSet.attrs['OpNames']:
mds += mdsOpsDataSet[op_index,b_index:e_index+1]
op_index += 1
mdsCPUGroup = fsFile['MDSCPUGroup']
mdsCPUDataSet = mdsCPUGroup['MDSCPUDataSet']
cpu = mdsCPUDataSet[b_index:e_index+1]
np.set_printoptions(threshold='nan')
#print "cpu: ", cpu
fig = plt.figure()
ax = fig.add_subplot(111)
Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], mds, 'g', label='metadata', Ave=True)
plt.xlabel('time')
plt.ylabel(r'$ops/sec$')
(handles, labels) = Graph.percent(ax, fsStepsDataSet[b_index:e_index+1], cpu, color='k', label='% CPU', Ave=True)
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
plt.title("%s %s Metadata Operations" % (fsStepsDataSet.attrs['day'],
fsStepsDataSet.attrs['fs']))
ax.set_ybound(lower = 0, upper = 120000)
if (not handles is None) and (not labels is None):
plt.legend(handles, labels)
if args.plot is None:
plt.show()
else:
plt.savefig(args.plot)
plt.cla()
return
def doMDSPlot(args, fsFile):
if args.progress == True:
print "MDS plot"
fsStepsGroup = fsFile['FSStepsGroup']
fsStepsDataSet = fsStepsGroup['FSStepsDataSet']
intervals = np.zeros(len(fsStepsDataSet))
intervals[1:] = np.diff(fsStepsDataSet)
fs=fsStepsDataSet.attrs['fs']
try:
host=fsStepsDataSet.attrs['host']
except:
host='hopper'
mds = np.zeros(len(fsStepsDataSet))
mdsOpsGroup = fsFile['MDSOpsGroup']
mdsOpsDataSet = mdsOpsGroup['MDSOpsDataSet']
op_index = 0
for op_name in mdsOpsDataSet.attrs['OpNames']:
mds += mdsOpsDataSet[op_index,:]
op_index += 1
highVals, = np.where(mds > 100000)
if len(highVals) > 0:
print "Warning: Exceedingly high values reported for ", highVals
print fsStepsDataSet[mds > 1000000]
print mds[mds > 1000000]
AggregateOps = np.sum(mds*intervals)
cpu = np.zeros(len(fsStepsDataSet))
mdsCPUGroup = fsFile['MDSCPUGroup']
mdsCPUDataSet = mdsCPUGroup['MDSCPUDataSet']
fig = plt.figure()
ax = fig.add_subplot(111)
Graph.timeSeries(ax, fsStepsDataSet, mds, 'b', label='mds', Ave=True)
plt.xlabel('time')
plt.ylabel(r'$Ops/sec$')
(handles, labels) = Graph.percent(ax, fsStepsDataSet, mdsCPUDataSet, color='k', label='% CPU', Ave=True)
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
plt.title("%s %s Metadata Operations" % (fsStepsDataSet.attrs['day'],
fsStepsDataSet.attrs['fs']))
ax.set_ybound(lower = 0, upper = 120000)
if (not handles is None) and (not labels is None):
plt.legend(handles, labels)
path = os.path.dirname(args.file)
if path == "":
path = "."
plt.savefig(path+'/'+host+'_'+fs+"_MDS+CPU.png")
plt.cla()
return(AggregateOps)
def do_action(args, bulk, brwfs):
steps = bulk.Steps.Steps[1:]
num_steps = len(steps)
times = bulk.Steps.Diff
# We don't want to distort the analysis by having some observation
# intervals longer than the nominal five seconds, do divide by the
# actual elapsed time to produce rates in MB/s, and Count/s
if not args.ost is None:
ost = bulk.getOST(ost=args.ost)
if ost is None:
print "we didn't find %s" % args.ost
readMBpS = ost.Read.Values[1:]
readMBpS /= (1024*1024)
writeMBpS = ost.Write.Values[1:]
writeMBpS /= (1024*1024)
# no cpu caclulation for one OST
cpu = np.zeros_like(readMBpS)
brwost = brwfs.getOST(ost=ost.name)
if brwost is None:
print "we didn't find %s among the brw_stats" % ost.name
return
id = brwost.getStatId(args.stat)
stat_bins = brwost.Bins[brwost.BrwIdDict[id]].Bins
readHistpS = np.diff(brwost.Read[id].Values)/times
readHistpS[np.where(readHistpS.mask==True)] = 0
writeHistpS = np.diff(brwost.Write[id].Values)/times
writeHistpS[np.where(writeHistpS.mask==True)] = 0
else:
readMBpS = bulk.Read.Values[1:]
readMBpS /= (1024*1024)
writeMBpS = bulk.Write.Values[1:]
writeMBpS /= (1024*1024)
cpu = bulk.CPU.Values[1:]
readHistpS = None
writeHistpS = None
num_osts = 0
np.set_printoptions(threshold='nan')
for oss in bulk.OSSs:
for ost in oss.OSTs:
print ost.name
brwost = brwfs.getOST(ost=ost.name)
id = brwost.getStatId(args.stat)
stat_bins = brwost.Bins[brwost.BrwIdDict[id]].Bins
readpS = np.diff(brwost.Read[id].Values)/times
readpS[np.where(readpS.mask==True)] = 0
if readHistpS is None:
readHistpS = np.zeros_like(readpS)
print readpS
readHistpS += readpS
writepS = np.diff(brwost.Write[id].Values)/times
writepS[np.where(writepS.mask==True)] = 0
if writeHistpS is None:
writeHistpS = np.zeros_like(writepS)
print writepS
writeHistpS += writepS
num_osts += 1
if (num_osts == 0) or (readHistpS is None) or (writeHistpS is None):
print "we didn't get anything for the brw_stats data"
return
readHistpS /= num_osts
writeHistpS /= num_osts
histSeries = np.transpose(np.vstack((readHistpS, writeHistpS)))
if args.report == True:
print "%d steps" % len(steps)
num_bins = len(stat_bins)
# Now we want to construct the two element distillation of histSeries
distill = np.zeros((2*num_bins, 4), dtype=np.float64)
distill[0:num_bins,0] = 1.0
distill[0:num_bins,1] = stat_bins
distill[num_bins:2*num_bins,2] = 1.0
distill[num_bins:2*num_bins,3] = stat_bins
A = np.matrix(histSeries)*np.matrix(distill)
# This is the result of the two_element_model.py calculation:
x = np.matrix([1.62589065e-03, 4.39766334e-09, 3.23092722e-03, 1.72900072e-09])
yhat = A * np.transpose(x)
if args.report == True:
print "yhat:"
print yhat
factor = 1
if args.hist == True:
if np.max(yhat) > 1.0:
factor = int(np.max(yhat)) + 1
utilHist, utilBins = np.histogram(yhat*100.0, bins=100, range=(0.0, 100.0*factor))
if args.report == True:
print "utilBins:"
print utilBins
print "utilHist:"
print utilHist
if args.plot == "noplot":
return
fig = plt.figure()
ax = fig.add_subplot(111)
Graph.timeSeries(ax, steps, readMBpS, 'r', label='read', Ave=False)
Graph.timeSeries(ax, steps, writeMBpS, 'b', label='write', Ave=False)
plt.xlabel('time')
plt.ylabel(r'$MiB/sec$')
(handles, labels) = Graph.percent(ax, steps, yhat*100.0, 'k', format="-", label='pct FSU', Ave=False)
if (not handles is None) and (not labels is None):
plt.legend(handles, labels)
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
dayStr = time.strftime("%Y-%m-%d", time.localtime(bulk.begin.sie))
plt.title("%s %s File System Utilization" % (dayStr, bulk.name))
if args.ybound is None:
ax.set_ybound(lower=0)
else:
ax.set_ybound(lower=0, upper=args.ybound)
if args.plot is None:
plt.show()
else:
plt.savefig(args.plot)
plt.cla()
if args.hist == False:
return
fig = plt.figure()
ax = fig.add_subplot(111)
ax.bar(utilBins[1:], utilHist, color='k')
plt.xlabel('pct fs util')
plt.ylabel('count')
plt.title('Distribution of File System Utilization')
if args.plot is None:
plt.show()
else:
plt.savefig('hist_'+args.plot)
plt.cla()
return
def doAction(args, b_sie, e_sie, fsFile):
fsStepsGroup = fsFile['FSStepsGroup']
fsStepsDataSet = fsStepsGroup['FSStepsDataSet']
if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]):
print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1])
return
if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]):
print "The ending timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1])
return
b_index = find_sie(b_sie, fsStepsDataSet)
e_index = find_sie(e_sie, fsStepsDataSet)
#print "data from index %d to %d" % (b_index, e_index)
fs=fsStepsDataSet.attrs['fs']
try:
host=fsStepsDataSet.attrs['host']
except:
host='hopper'
read = np.zeros(e_index - b_index + 1)
ostReadGroup = fsFile['OSTReadGroup']
ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet']
ost_index = 0
if args.report == True:
np.set_printoptions(threshold='nan')
for ost_name in ostBulkReadDataSet.attrs['OSTNames']:
#print "OST %s" % ost_name
read += ostBulkReadDataSet[ost_index,b_index:e_index+1]
#print "read:", ostBulkReadDataSet[ost_index,b_index:e_index+1]
ost_index += 1
read /= (1024*1024)
write = np.zeros(e_index - b_index + 1)
ostWriteGroup = fsFile['OSTWriteGroup']
ostBulkWriteDataSet = ostWriteGroup['OSTBulkWriteDataSet']
ost_index = 0
for ost_name in ostBulkWriteDataSet.attrs['OSTNames']:
#print "OST %s" % ost_name
write += ostBulkWriteDataSet[ost_index,b_index:e_index+1]
#print "write: ", ostBulkWriteDataSet[ost_index,b_index:e_index+1]
ost_index += 1
write /= (1024*1024)
cpu = np.zeros(e_index - b_index + 1)
ossCPUGroup = fsFile['OSSCPUGroup']
ossCPUDataSet = ossCPUGroup['OSSCPUDataSet']
oss_index = 0
for oss_name in ossCPUDataSet.attrs['OSSNames']:
cpu += ossCPUDataSet[oss_index,b_index:e_index+1]
oss_index += 1
cpu /= oss_index
if args.report == True:
print "read:", read
print "write: ", write
print "cpu: ", cpu
if args.plot == "noplot":
return
fig = plt.figure()
ax = fig.add_subplot(111)
Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], read, 'r', label='read', Ave=args.ave)
Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], write, 'b', label='write', Ave=args.ave)
(handles, labels) = ax.get_legend_handles_labels()
plt.xlabel('time')
plt.ylabel(r'$MiB/sec$')
if args.cpu == True:
(handles, labels) = Graph.percent(ax, fsStepsDataSet[b_index:e_index+1], cpu, color='k', label='% CPU', Ave=args.ave)
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
plt.title("%s %s aggregate I/O" % (fsStepsDataSet.attrs['day'],
fsStepsDataSet.attrs['fs']))
if args.ybound is None:
ax.set_ybound(lower = 0, upper = 50000)
else:
ax.set_ybound(lower = 0, upper = args.ybound)
if (not handles is None) and (not labels is None):
plt.legend(handles, labels)
if args.plot is None:
plt.show()
else:
plt.savefig(args.plot)
plt.cla()
return
def do_action(args, b_sie, e_sie, fsFile):
fsStepsGroup = fsFile['FSStepsGroup']
fsStepsDataSet = fsStepsGroup['FSStepsDataSet']
if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]):
print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1])
return
if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]):
print "The ending timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1])
return
b_index = find_sie(b_sie, fsStepsDataSet)
e_index = find_sie(e_sie, fsStepsDataSet)
if b_index == None:
print "brw_stats_model_h5lmt.do_action(): Failed to find timestamp index for %d" % b_sie
return
if e_index == None:
print "brw_stats_model_h5lmt.do_action(): Failed to find timestamp index for %d" % e_sie
return
fs=fsStepsDataSet.attrs['fs']
try:
host=fsStepsDataSet.attrs['host']
except:
host='hopper'
readMBpS = np.zeros(e_index - b_index + 1)
ostReadGroup = fsFile['OSTReadGroup']
ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet']
ost_index = 0
for ost_name in ostBulkReadDataSet.attrs['OSTNames']:
readMBpS += ostBulkReadDataSet[ost_index,b_index:e_index+1]
ost_index += 1
readMBpS /= 1024*1024
np.set_printoptions(threshold='nan')
writeMBpS = np.zeros(e_index - b_index + 1)
ostWriteGroup = fsFile['OSTWriteGroup']
ostBulkWriteDataSet = ostWriteGroup['OSTBulkWriteDataSet']
ost_index = 0
for ost_name in ostBulkWriteDataSet.attrs['OSTNames']:
writeMBpS += ostBulkWriteDataSet[ost_index,b_index:e_index+1]
ost_index += 1
writeMBpS /= 1024*1024
ostIosizeReadDataSet = ostReadGroup['OSTIosizeReadDataSet']
if args.stat != ostIosizeReadDataSet.attrs['stat']:
print "We should only be seeing BRW_IOSIZE statistics not %s" % ostIosizeReadDataSet.attrs['stat']
return
bins = ostIosizeReadDataSet.attrs['bins']
ostIosizeWriteDataSet = ostWriteGroup['OSTIosizeWriteDataSet']
readHistpS = None
writeHistpS = None
ost_index = 0
if args.report == True:
np.set_printoptions(threshold='nan')
for ost_name in ostBulkReadDataSet.attrs['OSTNames']:
if args.progress == True:
print "OST %d: %s" % (ost_index, ost_name)
readpS = ostIosizeReadDataSet[ost_index,:,b_index:e_index+1]
if readHistpS is None:
readHistpS = np.zeros_like(readpS)
#print readpS
readHistpS += readpS
writepS = ostIosizeWriteDataSet[ost_index,:,b_index:e_index+1]
if writeHistpS is None:
writeHistpS = np.zeros_like(writepS)
#print writepS
writeHistpS += writepS
ost_index += 1
if (ost_index == 0) or (readHistpS is None) or (writeHistpS is None):
print "we didn't get anything for the brw_stats data"
return
readHistpS /= ost_index
writeHistpS /= ost_index
histSeries = np.transpose(np.vstack((readHistpS, writeHistpS)))
if args.report == True:
print "%d steps" % len(fsStepsDataSet)
num_bins = len(bins)
# Now we want to construct the two element distillation of histSeries
distill = np.zeros((2*num_bins, 4), dtype=np.float64)
distill[0:num_bins,0] = 1.0
distill[0:num_bins,1] = bins
distill[num_bins:2*num_bins,2] = 1.0
distill[num_bins:2*num_bins,3] = bins
A = np.matrix(histSeries)*np.matrix(distill)
# This is the result of the two_element_model.py calculation:
x = np.matrix([1.62589065e-03, 4.39766334e-09, 3.23092722e-03, 1.72900072e-09])
yhat = A * np.transpose(x)
yhat = np.array(yhat)
yhat *= 100.0
if args.report == True:
print "yhat (percent):"
print yhat
if args.hist == True:
doHist(args, histSeries, yhat, fsStepsDataSet)
if args.plot == "noplot":
return
fig = plt.figure()
ax = fig.add_subplot(111)
Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], readMBpS, 'r', label='read', Ave=False)
Graph.timeSeries(ax, fsStepsDataSet[b_index:e_index+1], writeMBpS, 'b', label='write', Ave=False)
plt.xlabel('time')
plt.ylabel(r'$MiB/sec$')
(handles, labels) = Graph.percent(ax, fsStepsDataSet[b_index:e_index+1], yhat, 'k', label='FSU', Ave=False, ybound=200.0)
if (not handles is None) and (not labels is None):
plt.legend(handles, labels)
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
dayStr = time.strftime("%Y-%m-%d", time.localtime(fsStepsDataSet[0]))
plt.title("%s %s File System Utilization" % (dayStr, fs))
if args.ybound is None:
ax.set_ybound(lower=0, upper=50000)
else:
ax.set_ybound(lower=0, upper=args.ybound)
if args.plot is None:
plt.show()
else:
plt.savefig(args.plot)
return
def do_plot(B, mode=None, plot=None, ybound=None,
scale=1024.0*1024.0, withCPU=True):
if args.lines == True:
format = '-'
else:
format = None
fig = plt.figure()
ax = fig.add_subplot(111)
steps = B.Steps.Steps
ymax = 0
np.set_printoptions(threshold='nan')
if mode == 'Both':
values = B.Bulk.Values/scale
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'b', label='read and write',
Ave=True, format=format)
elif mode is None:
values = B.Read.Values/scale
#print "read: ", values
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'r', label='read',
Ave=True, format=format)
values = B.Write.Values/scale
#print "write: ", values
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'b', label='write',
Ave=True, format=format)
elif mode == 'Read':
values = B.Read.Values/scale
#print "read: ", values
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'r', label='read',
Ave=True, format=format)
else:
values = B.Write.Values/scale
#print "write: ", values
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'b', label='write',
Ave=True, format=format)
plt.xlabel('time')
plt.ylabel(r'$MiB/sec$')
if (withCPU == True) and (not B.CPU is None):
values = np.array(B.CPU.Values)
#values[np.where(values.mask==True)] = 0.0
(handles, labels) = Graph.percent(ax, steps, values, 'k',
label='pct CPU', Ave=True)
# insert bogus here for testing
if (not handles is None) and (not labels is None):
plt.legend(handles, labels)
else:
print "bulk.do_plot(): Warning - Plotting CPU utilization failed."
else:
plt.legend()
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
dayStr = time.strftime("%Y-%m-%d", time.localtime(B.begin.sie))
plt.title("%s %s aggregate I/O" % (dayStr, B.name))
if ybound is None:
ybound = ymax
ax.set_ybound(lower = 0, upper = ybound)
if plot is None:
plt.show()
else:
plt.savefig(plot)
plt.cla()
def do_plot(oss, mode=None, plot=None, ybound=None,
scale=1024.0*1024.0):
if args.lines == True:
format = '-'
else:
format = None
fig = plt.figure()
ax = fig.add_subplot(111)
steps = oss.Steps.Steps
ymax = 0
if mode == 'Both':
values = oss.OSS.Values/scale
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'b', label='read and write',
Ave=True, format=format)
elif mode is None:
values = oss.Read.Values/scale
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'r', label='read',
Ave=True, format=format)
values = oss.Write.Values/scale
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'b', label='write',
Ave=True, format=format)
elif mode == 'Read':
values = oss.Read.Values/scale
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'r', label='read',
Ave=True, format=format)
else:
values = oss.Write.Values/scale
max = np.max(values)
if max > ymax:
ymax = max
Graph.timeSeries(ax, steps, values, 'b', label='write',
Ave=True)
plt.xlabel('time')
plt.ylabel(r'$MiB/sec$')
if not oss.CPU is None:
values = oss.CPU.Values
(handles, labels) = Graph.percent(ax, steps, values, 'k',
label='% CPU', Ave=True)
if (not handles is None) and (not labels is None):
plt.legend(handles, labels)
else:
print "oss.do_plot(): Warning - Plotting CPU utilization failed."
else:
plt.legend()
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
dayStr = time.strftime("%Y-%m-%d", time.localtime(oss.begin.sie))
plt.title("%s %s aggregate I/O" % (dayStr, oss.name))
if ybound is None:
ybound = ymax
ax.set_ybound(lower = 0, upper = ybound)
if plot is None:
plt.show()
else:
plt.savefig(plot)
plt.cla()
def doRatePlot(args, fsFile):
"""
The values in the h5lmt arrays are time series of observaitons
of true rates in in MB/s. In order to aggregate you need to
multiply by the interval lengths. Those are hard coded to
five seconds in the current implementation, but that is not
something to necessarily rely on going forward.
"""
if args.progress == True:
print "Rate plot"
fsStepsGroup = fsFile['FSStepsGroup']
fsStepsDataSet = fsStepsGroup['FSStepsDataSet']
intervals = np.zeros(len(fsStepsDataSet))
intervals[1:] = np.diff(fsStepsDataSet)
fs=fsStepsDataSet.attrs['fs']
try:
host=fsStepsDataSet.attrs['host']
except:
host='hopper'
read = np.zeros(len(fsStepsDataSet))
ostReadGroup = fsFile['OSTReadGroup']
ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet']
ost_index = 0
for ost_name in ostBulkReadDataSet.attrs['OSTNames']:
read += ostBulkReadDataSet[ost_index,:]
ost_index += 1
read /= (1024*1024)
AggregateRead = np.sum(read*intervals)
write = np.zeros(len(fsStepsDataSet))
ostWriteGroup = fsFile['OSTWriteGroup']
ostBulkWriteDataSet = ostWriteGroup['OSTBulkWriteDataSet']
ost_index = 0
for ost_name in ostBulkWriteDataSet.attrs['OSTNames']:
write += ostBulkWriteDataSet[ost_index,:]
ost_index += 1
write /= (1024*1024)
AggregateWrite = np.sum(write*intervals)
cpu = np.zeros(len(fsStepsDataSet))
ossCPUGroup = fsFile['OSSCPUGroup']
ossCPUDataSet = ossCPUGroup['OSSCPUDataSet']
oss_index = 0
for oss_name in ossCPUDataSet.attrs['OSSNames']:
cpu += ossCPUDataSet[oss_index,:]
oss_index += 1
cpu /= oss_index
fig = plt.figure()
ax = fig.add_subplot(111)
Graph.timeSeries(ax, fsStepsDataSet, read, 'r', label='read', Ave=True)
Graph.timeSeries(ax, fsStepsDataSet, write, 'b', label='write', Ave=True)
plt.xlabel('time')
plt.ylabel(r'$MiB/sec$')
(handles, labels) = Graph.percent(ax, fsStepsDataSet, cpu, color='k', label='% CPU', Ave=True)
plt.setp( ax.get_xticklabels(), rotation=30, horizontalalignment='right')
plt.title("%s %s aggregate I/O" % (fsStepsDataSet.attrs['day'],
fsStepsDataSet.attrs['fs']))
ax.set_ybound(lower = 0, upper = 50000)
if (not handles is None) and (not labels is None):
plt.legend(handles, labels)
else:
plt.legend()
path = os.path.dirname(args.file)
if path == "":
path = "."
plt.savefig(path+'/'+host+'_'+fs+"_bulkRateCPU.png")
plt.cla()
return(AggregateRead, AggregateWrite)
