def doAction(args, b_sie, e_sie, fsFile):
fsStepsGroup = fsFile['FSStepsGroup']
fsStepsDataSet = fsStepsGroup['FSStepsDataSet']
if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]):
print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1])
return
if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]):
print "The ending timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1])
return
b_index = find_sie(b_sie, fsStepsDataSet)
e_index = find_sie(e_sie, fsStepsDataSet)
if args.report == True:
print "data from index %d to %d" % (b_index, e_index)
fs=fsStepsDataSet.attrs['fs']
try:
host=fsStepsDataSet.attrs['host']
except:
host='hopper'
ostReadGroup = fsFile['OSTReadGroup']
ostWriteGroup = fsFile['OSTWriteGroup']
ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet']
ostIosizeReadDataSet = ostReadGroup['OSTIosizeReadDataSet']
if args.stat != ostIosizeReadDataSet.attrs['stat']:
print "We should only be seeing BRW_IOSIZE statistics not %s" % ostIosizeReadDataSet.attrs['stat']
return
bins = ostIosizeReadDataSet.attrs['bins']
ostIosizeWriteDataSet = ostWriteGroup['OSTIosizeWriteDataSet']
readHistpS = None
writeHistpS = None
ost_index = 0
foundOST = False
for ost_name in ostBulkReadDataSet.attrs['OSTNames']:
if args.progress == True:
print "OST %d: %s" % (ost_index, ost_name)
if not args.ost is None:
if args.ost == ost_name:
foundOST = True
else:
ost_index += 1
continue
readpS = ostIosizeReadDataSet[ost_index,:,b_index:e_index+1]
if readHistpS is None:
readHistpS = np.zeros_like(readpS)
#print readpS
readHistpS += readpS
writepS = ostIosizeWriteDataSet[ost_index,:,b_index:e_index+1]
if writeHistpS is None:
writeHistpS = np.zeros_like(writepS)
#print writepS
writeHistpS += writepS
ost_index += 1
if (not args.ost is None) and (foundOST == False):
print "%s not found" % args.ost
return
if (ost_index == 0) or (readHistpS is None) or (writeHistpS is None):
print "we didn't get anything for the brw_stats data"
return
readIosizeHist = np.sum(readHistpS, axis=1)
writeIosizeHist = np.sum(writeHistpS, axis=1)
if not args.save is None:
fsFile = h5py.File(args.save, 'a')
try:
fsIosizeGroup = fsFile["FSIosizeGroup"]
except KeyError:
fsIosizeGroup = fsFile.create_group("FSIosizeGroup")
try:
fsIosizeBinsDataSet = fsIosizeGroup["FSIosizeBinsDataSet"]
fsIosizeBinsDataSet = bins
except KeyError:
fsIosizeBinsDataSet = fsIosizeGroup.create_dataset("FSIosizeBinsDataSet", data=bins)
try:
fsIosizeReadDataSet = fsIosizeGroup["FSIosizeReadDataSet"]
fsIosizeReadDataSet = readIosizeHist
except KeyError:
fsIosizeReadDataSet = fsIosizeGroup.create_dataset("FSIosizeReadDataSet", data=readIosizeHist)
try:
fsIosizeWriteDataSet = fsIosizeGroup["FSIosizeWriteDataSet"]
fsIosizeWriteDataSet = writeIosizeHist
except:
fsIosizeWriteDataSet = fsIosizeGroup.create_dataset("FSIosizeWriteDataSet", data=writeIosizeHist)
fsIosizeGroup.attrs['day'] = fsStepsDataSet.attrs['day']
fsIosizeGroup.attrs['nextday'] = fsStepsDataSet.attrs['nextday']
fsIosizeGroup.attrs['host'] = fsStepsDataSet.attrs['host']
fsIosizeGroup.attrs['fs'] = fsStepsDataSet.attrs['fs']
fsIosizeGroup.attrs['stat'] = ostIosizeReadDataSet.attrs['stat']
fsIosizeGroup.attrs['bins'] = ostIosizeReadDataSet.attrs['bins']
fsFile.close()
if args.plot == "noplot":
return
fig = plt.figure()
fig.subplots_adjust(bottom=0.2)
ax = fig.add_subplot(111)
width = 0.35
x = np.arange(len(bins))
offset=0.0
Graph.bar(ax, x, readIosizeHist, width=width, offset=offset, color='r', label='read')
offset += width
Graph.bar(ax, x, writeIosizeHist, width=width, offset=offset, color='b', label='write')
offset += width
ax.set_ylabel('Count')
ax.set_xlabel('MB')
ax.set_title('%s %s I/O size histogram' % (host, fs) )
ax.set_xticks(x+width)
ax.set_xticklabels( bins, rotation=45, horizontalalignment='right' )
ax.legend()
if args.plot is None:
plt.show()
else:
plt.savefig( args.plot )
plt.cla()
return
def doAction(args, fsFile, addFile):
if not addFile is None:
try:
addIosizeGroup = addFile["FSIosizeGroup"]
except KeyError:
print "fsiosize.py.doAction(): no FSIosizeGroup in %s. Aborting" % args.add
return
try:
addIosizeBinsDataSet = addIosizeGroup["FSIosizeBinsDataSet"]
except KeyError:
print "fsiosize.py.doAction(): no FSIosizeBinsDataSet in %s. Aborting" % args.add
return
try:
addIosizeReadDataSet = addIosizeGroup["FSIosizeReadDataSet"]
except KeyError:
print "fsiosize.py.doAction(): no FSIosizeReadDataSet in %s. Aborting" % args.add
return
try:
addIosizeWriteDataSet = addIosizeGroup["FSIosizeWriteDataSet"]
except:
print "fsiosize.py.doAction(): no FSIosizeWriteDataSet in %s. Aborting" % args.add
return
for attr in ('day', 'nextday', 'host', 'fs', 'stat', 'bins'):
if not attr in addIosizeGroup.attrs:
print "fsiosize.py.doAction(): The '%s' attribute is not in the %s fsIosizeGroup" % (attr, args.add)
return
try:
fsIosizeGroup = fsFile["FSIosizeGroup"]
except KeyError:
fsIosizeGroup = fsFile.create_group("FSIosizeGroup")
try:
fsIosizeBinsDataSet = fsIosizeGroup["FSIosizeBinsDataSet"]
for bin in fsIosizeBinsDataSet:
if not bin in addIosizeBinsDataSet:
print "fsiosize.py.doAction(): %f in %s but not %s. Aborting" % (bin, args.file, args.add)
return
for bin in addIosizeBinsDataSet:
if not bin in fsIosizeBinsDataSet:
print "fsiosize.py.doAction(): %f in %s but not %s. Aborting" % (bin, args.add, args.file)
return
except KeyError:
fsIosizeBinsDataSet = fsIosizeGroup.create_dataset("FSIosizeBinsDataSet", data=addIosizeBinsDataSet)
try:
fsIosizeReadDataSet = fsIosizeGroup["FSIosizeReadDataSet"]
fsIosizeReadDataSet[:] += addIosizeReadDataSet[:]
except KeyError:
fsIosizeReadDataSet = fsIosizeGroup.create_dataset("FSIosizeReadDataSet", data=addIosizeReadDataSet)
try:
fsIosizeWriteDataSet = fsIosizeGroup["FSIosizeWriteDataSet"]
fsIosizeWriteDataSet[:] += addIosizeWriteDataSet[:]
except:
fsIosizeWriteDataSet = fsIosizeGroup.create_dataset("FSIosizeWriteDataSet", data=addIosizeWriteDataSet)
if 'day' in fsIosizeGroup.attrs:
if fsIosizeGroup.attrs['day'] > addIosizeGroup.attrs['day']:
fsIosizeGroup.attrs['day'] = addIosizeGroup.attrs['day']
else:
fsIosizeGroup.attrs['day'] = addIosizeGroup.attrs['day']
if 'nextday' in fsIosizeGroup.attrs:
if fsIosizeGroup.attrs['nextday'] < addIosizeGroup.attrs['nextday']:
fsIosizeGroup.attrs['nextday'] = addIosizeGroup.attrs['nextday']
else:
fsIosizeGroup.attrs['nextday'] = addIosizeGroup.attrs['nextday']
for attr in ('host', 'fs', 'stat'):
if (attr in fsIosizeGroup.attrs) and not (fsIosizeGroup.attrs[attr] == addIosizeGroup.attrs[attr]):
print "fsiosize.py.doAction(): %s does not match between %s and %s. Aborting" % (attr, args.add, args.file)
return
else:
fsIosizeGroup.attrs[attr] = addIosizeGroup.attrs[attr]
if 'bins' in fsIosizeGroup.attrs:
for bin in fsIosizeGroup.attrs['bins']:
if not bin in addIosizeGroup.attrs['bins']:
print "fsiosize.py.doAction(): bin %s in %s but not %s. Aborting" % (bin, args.file, args.add)
return
for bin in addIosizeGroup.attrs['bins']:
if not bin in fsIosizeGroup.attrs['bins']:
print "fsiosize.py.doAction(): bin %s in %s but not %s. Aborting" % (bin, args.add, args.file)
return
else:
fsIosizeGroup.attrs['bins'] = addIosizeGroup.attrs['bins']
# end of accumulating args.add into args.file
else:
try:
fsIosizeGroup = fsFile["FSIosizeGroup"]
except KeyError:
print "fsiosize.py.doAction(): no FSIosizeGroup in %s. Aborting" % args.file
return
try:
fsIosizeBinsDataSet = fsIosizeGroup["FSIosizeBinsDataSet"]
except KeyError:
print "fsiosize.py.doAction(): no FSIosizeBinsDataSet in %s. Aborting" % args.file
return
try:
fsIosizeReadDataSet = fsIosizeGroup["FSIosizeReadDataSet"]
except KeyError:
print "fsiosize.py.doAction(): no FSIosizeReadDataSet in %s. Aborting" % args.file
return
try:
fsIosizeWriteDataSet = fsIosizeGroup["FSIosizeWriteDataSet"]
except:
print "fsiosize.py.doAction(): no FSIosizeWriteDataSet in %s. Aborting" % args.file
return
for attr in ('day', 'nextday', 'host', 'fs', 'stat', 'bins'):
if not attr in fsIosizeGroup.attrs:
print "fsiosize.py.doAction(): The '%s' attribute is not in the %s fsIosizeGroup" % (attr, args.file)
return
if args.report == True:
np.set_printoptions(threshold='nan')
for attr in ('day', 'nextday', 'host', 'fs', 'stat', 'bins'):
print "%s = %s" % (attr, fsIosizeGroup.attrs[attr])
print 'bins:', fsIosizeBinsDataSet[:]
print 'read:', fsIosizeReadDataSet[:]
print 'write', fsIosizeWriteDataSet[:]
if args.plot == "noplot":
return
bins = fsIosizeGroup.attrs['bins']
host = fsIosizeGroup.attrs['host']
fs = fsIosizeGroup.attrs['fs']
day = fsIosizeGroup.attrs['day']
nextday = fsIosizeGroup.attrs['nextday']
fig = plt.figure()
fig.subplots_adjust(bottom=0.2)
ax = fig.add_subplot(111)
width = 0.35
x = np.arange(len(bins))
offset=0.0
Graph.bar(ax, x, fsIosizeReadDataSet, width=width, offset=offset, color='r', label='read')
offset += width
Graph.bar(ax, x, fsIosizeWriteDataSet, width=width, offset=offset, color='b', label='write')
offset += width
ax.set_ylabel('Count')
ax.set_xlabel('Bytes')
ax.set_title('%s %s I/O sizes, %s to %s' % (host, fs, day, nextday) )
ax.set_xticks(x+width)
ax.set_xticklabels(bins, rotation=45, horizontalalignment='right' )
ax.legend()
if args.plot is None:
plt.show()
else:
plt.savefig( args.plot )
plt.cla()
return
