def doAction(args, b_sie, e_sie, fsFile): fsStepsGroup = fsFile['FSStepsGroup'] fsStepsDataSet = fsStepsGroup['FSStepsDataSet'] if (b_sie < fsStepsDataSet[0]) or (b_sie > fsStepsDataSet[-1]): print "The beginning timestamp %d is outside the date range from %d to %d" % (b_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return if (e_sie < fsStepsDataSet[0]) or (e_sie > fsStepsDataSet[-1]): print "The ending timestamp %d is outside the date range from %d to %d" % (e_sie, fsStepsDataSet[0], fsStepsDataSet[-1]) return b_index = find_sie(b_sie, fsStepsDataSet) e_index = find_sie(e_sie, fsStepsDataSet) if args.report == True: print "data from index %d to %d" % (b_index, e_index) fs=fsStepsDataSet.attrs['fs'] try: host=fsStepsDataSet.attrs['host'] except: host='hopper' ostReadGroup = fsFile['OSTReadGroup'] ostWriteGroup = fsFile['OSTWriteGroup'] ostBulkReadDataSet = ostReadGroup['OSTBulkReadDataSet'] ostIosizeReadDataSet = ostReadGroup['OSTIosizeReadDataSet'] if args.stat != ostIosizeReadDataSet.attrs['stat']: print "We should only be seeing BRW_IOSIZE statistics not %s" % ostIosizeReadDataSet.attrs['stat'] return bins = ostIosizeReadDataSet.attrs['bins'] ostIosizeWriteDataSet = ostWriteGroup['OSTIosizeWriteDataSet'] readHistpS = None writeHistpS = None ost_index = 0 foundOST = False for ost_name in ostBulkReadDataSet.attrs['OSTNames']: if args.progress == True: print "OST %d: %s" % (ost_index, ost_name) if not args.ost is None: if args.ost == ost_name: foundOST = True else: ost_index += 1 continue readpS = ostIosizeReadDataSet[ost_index,:,b_index:e_index+1] if readHistpS is None: readHistpS = np.zeros_like(readpS) #print readpS readHistpS += readpS writepS = ostIosizeWriteDataSet[ost_index,:,b_index:e_index+1] if writeHistpS is None: writeHistpS = np.zeros_like(writepS) #print writepS writeHistpS += writepS ost_index += 1 if (not args.ost is None) and (foundOST == False): print "%s not found" % args.ost return if (ost_index == 0) or (readHistpS is None) or (writeHistpS is None): print "we didn't get anything for the brw_stats data" return readIosizeHist = np.sum(readHistpS, axis=1) writeIosizeHist = np.sum(writeHistpS, axis=1) if not args.save is None: fsFile = h5py.File(args.save, 'a') try: fsIosizeGroup = fsFile["FSIosizeGroup"] except KeyError: fsIosizeGroup = fsFile.create_group("FSIosizeGroup") try: fsIosizeBinsDataSet = fsIosizeGroup["FSIosizeBinsDataSet"] fsIosizeBinsDataSet = bins except KeyError: fsIosizeBinsDataSet = fsIosizeGroup.create_dataset("FSIosizeBinsDataSet", data=bins) try: fsIosizeReadDataSet = fsIosizeGroup["FSIosizeReadDataSet"] fsIosizeReadDataSet = readIosizeHist except KeyError: fsIosizeReadDataSet = fsIosizeGroup.create_dataset("FSIosizeReadDataSet", data=readIosizeHist) try: fsIosizeWriteDataSet = fsIosizeGroup["FSIosizeWriteDataSet"] fsIosizeWriteDataSet = writeIosizeHist except: fsIosizeWriteDataSet = fsIosizeGroup.create_dataset("FSIosizeWriteDataSet", data=writeIosizeHist) fsIosizeGroup.attrs['day'] = fsStepsDataSet.attrs['day'] fsIosizeGroup.attrs['nextday'] = fsStepsDataSet.attrs['nextday'] fsIosizeGroup.attrs['host'] = fsStepsDataSet.attrs['host'] fsIosizeGroup.attrs['fs'] = fsStepsDataSet.attrs['fs'] fsIosizeGroup.attrs['stat'] = ostIosizeReadDataSet.attrs['stat'] fsIosizeGroup.attrs['bins'] = ostIosizeReadDataSet.attrs['bins'] fsFile.close() if args.plot == "noplot": return fig = plt.figure() fig.subplots_adjust(bottom=0.2) ax = fig.add_subplot(111) width = 0.35 x = np.arange(len(bins)) offset=0.0 Graph.bar(ax, x, readIosizeHist, width=width, offset=offset, color='r', label='read') offset += width Graph.bar(ax, x, writeIosizeHist, width=width, offset=offset, color='b', label='write') offset += width ax.set_ylabel('Count') ax.set_xlabel('MB') ax.set_title('%s %s I/O size histogram' % (host, fs) ) ax.set_xticks(x+width) ax.set_xticklabels( bins, rotation=45, horizontalalignment='right' ) ax.legend() if args.plot is None: plt.show() else: plt.savefig( args.plot ) plt.cla() return
def doAction(args, fsFile, addFile): if not addFile is None: try: addIosizeGroup = addFile["FSIosizeGroup"] except KeyError: print "fsiosize.py.doAction(): no FSIosizeGroup in %s. Aborting" % args.add return try: addIosizeBinsDataSet = addIosizeGroup["FSIosizeBinsDataSet"] except KeyError: print "fsiosize.py.doAction(): no FSIosizeBinsDataSet in %s. Aborting" % args.add return try: addIosizeReadDataSet = addIosizeGroup["FSIosizeReadDataSet"] except KeyError: print "fsiosize.py.doAction(): no FSIosizeReadDataSet in %s. Aborting" % args.add return try: addIosizeWriteDataSet = addIosizeGroup["FSIosizeWriteDataSet"] except: print "fsiosize.py.doAction(): no FSIosizeWriteDataSet in %s. Aborting" % args.add return for attr in ('day', 'nextday', 'host', 'fs', 'stat', 'bins'): if not attr in addIosizeGroup.attrs: print "fsiosize.py.doAction(): The '%s' attribute is not in the %s fsIosizeGroup" % (attr, args.add) return try: fsIosizeGroup = fsFile["FSIosizeGroup"] except KeyError: fsIosizeGroup = fsFile.create_group("FSIosizeGroup") try: fsIosizeBinsDataSet = fsIosizeGroup["FSIosizeBinsDataSet"] for bin in fsIosizeBinsDataSet: if not bin in addIosizeBinsDataSet: print "fsiosize.py.doAction(): %f in %s but not %s. Aborting" % (bin, args.file, args.add) return for bin in addIosizeBinsDataSet: if not bin in fsIosizeBinsDataSet: print "fsiosize.py.doAction(): %f in %s but not %s. Aborting" % (bin, args.add, args.file) return except KeyError: fsIosizeBinsDataSet = fsIosizeGroup.create_dataset("FSIosizeBinsDataSet", data=addIosizeBinsDataSet) try: fsIosizeReadDataSet = fsIosizeGroup["FSIosizeReadDataSet"] fsIosizeReadDataSet[:] += addIosizeReadDataSet[:] except KeyError: fsIosizeReadDataSet = fsIosizeGroup.create_dataset("FSIosizeReadDataSet", data=addIosizeReadDataSet) try: fsIosizeWriteDataSet = fsIosizeGroup["FSIosizeWriteDataSet"] fsIosizeWriteDataSet[:] += addIosizeWriteDataSet[:] except: fsIosizeWriteDataSet = fsIosizeGroup.create_dataset("FSIosizeWriteDataSet", data=addIosizeWriteDataSet) if 'day' in fsIosizeGroup.attrs: if fsIosizeGroup.attrs['day'] > addIosizeGroup.attrs['day']: fsIosizeGroup.attrs['day'] = addIosizeGroup.attrs['day'] else: fsIosizeGroup.attrs['day'] = addIosizeGroup.attrs['day'] if 'nextday' in fsIosizeGroup.attrs: if fsIosizeGroup.attrs['nextday'] < addIosizeGroup.attrs['nextday']: fsIosizeGroup.attrs['nextday'] = addIosizeGroup.attrs['nextday'] else: fsIosizeGroup.attrs['nextday'] = addIosizeGroup.attrs['nextday'] for attr in ('host', 'fs', 'stat'): if (attr in fsIosizeGroup.attrs) and not (fsIosizeGroup.attrs[attr] == addIosizeGroup.attrs[attr]): print "fsiosize.py.doAction(): %s does not match between %s and %s. Aborting" % (attr, args.add, args.file) return else: fsIosizeGroup.attrs[attr] = addIosizeGroup.attrs[attr] if 'bins' in fsIosizeGroup.attrs: for bin in fsIosizeGroup.attrs['bins']: if not bin in addIosizeGroup.attrs['bins']: print "fsiosize.py.doAction(): bin %s in %s but not %s. Aborting" % (bin, args.file, args.add) return for bin in addIosizeGroup.attrs['bins']: if not bin in fsIosizeGroup.attrs['bins']: print "fsiosize.py.doAction(): bin %s in %s but not %s. Aborting" % (bin, args.add, args.file) return else: fsIosizeGroup.attrs['bins'] = addIosizeGroup.attrs['bins'] # end of accumulating args.add into args.file else: try: fsIosizeGroup = fsFile["FSIosizeGroup"] except KeyError: print "fsiosize.py.doAction(): no FSIosizeGroup in %s. Aborting" % args.file return try: fsIosizeBinsDataSet = fsIosizeGroup["FSIosizeBinsDataSet"] except KeyError: print "fsiosize.py.doAction(): no FSIosizeBinsDataSet in %s. Aborting" % args.file return try: fsIosizeReadDataSet = fsIosizeGroup["FSIosizeReadDataSet"] except KeyError: print "fsiosize.py.doAction(): no FSIosizeReadDataSet in %s. Aborting" % args.file return try: fsIosizeWriteDataSet = fsIosizeGroup["FSIosizeWriteDataSet"] except: print "fsiosize.py.doAction(): no FSIosizeWriteDataSet in %s. Aborting" % args.file return for attr in ('day', 'nextday', 'host', 'fs', 'stat', 'bins'): if not attr in fsIosizeGroup.attrs: print "fsiosize.py.doAction(): The '%s' attribute is not in the %s fsIosizeGroup" % (attr, args.file) return if args.report == True: np.set_printoptions(threshold='nan') for attr in ('day', 'nextday', 'host', 'fs', 'stat', 'bins'): print "%s = %s" % (attr, fsIosizeGroup.attrs[attr]) print 'bins:', fsIosizeBinsDataSet[:] print 'read:', fsIosizeReadDataSet[:] print 'write', fsIosizeWriteDataSet[:] if args.plot == "noplot": return bins = fsIosizeGroup.attrs['bins'] host = fsIosizeGroup.attrs['host'] fs = fsIosizeGroup.attrs['fs'] day = fsIosizeGroup.attrs['day'] nextday = fsIosizeGroup.attrs['nextday'] fig = plt.figure() fig.subplots_adjust(bottom=0.2) ax = fig.add_subplot(111) width = 0.35 x = np.arange(len(bins)) offset=0.0 Graph.bar(ax, x, fsIosizeReadDataSet, width=width, offset=offset, color='r', label='read') offset += width Graph.bar(ax, x, fsIosizeWriteDataSet, width=width, offset=offset, color='b', label='write') offset += width ax.set_ylabel('Count') ax.set_xlabel('Bytes') ax.set_title('%s %s I/O sizes, %s to %s' % (host, fs, day, nextday) ) ax.set_xticks(x+width) ax.set_xticklabels(bins, rotation=45, horizontalalignment='right' ) ax.legend() if args.plot is None: plt.show() else: plt.savefig( args.plot ) plt.cla() return