def findBlockIOusage(execHostStr, jobStart, jobEnd):
global allJobIORead
global allJobIOWrite
hostList = []
for host in execHostStr.split('+'):
hostList.append(host.split('/')[0])
hostList = list(set(hostList))
readBytes = 0.0
writeBytes = 0.0
peakReadBytes = 0.0
peakWriteBytes = 0.0
flag = False
for host in hostList:
nodeNumber = int((host.split('-')[1])[1:])
readSectorsPer10mins = taccExtract.extractTaccStats(taccDB,\
nodeNumber, 'block', 'rd_sectors,E,U=512B',\
int(jobStart), int(jobEnd))
writeSectorsPer10mins = taccExtract.extractTaccStats(taccDB,\
nodeNumber, 'block', 'wr_sectors,E,U=512B',\
int(jobStart), int(jobEnd))
allJobIORead.extend(np.diff(readSectorsPer10mins))
allJobIOWrite.extend(np.diff(writeSectorsPer10mins))
if len(readSectorsPer10mins) > 0:
totalRead = readSectorsPer10mins[-1] - readSectorsPer10mins[0]
if len(readSectorsPer10mins) > 1:
if peakReadBytes < np.amax(np.diff(readSectorsPer10mins)):
peakReadBytes = np.amax(np.diff(readSectorsPer10mins))
else:
totalRead = 0
if len(writeSectorsPer10mins) > 0:
totalWrite = writeSectorsPer10mins[-1] - writeSectorsPer10mins[0]
if len(writeSectorsPer10mins) > 1:
if (peakWriteBytes < np.amax(np.diff(writeSectorsPer10mins))):
peakWriteBytes = np.amax(np.diff(writeSectorsPer10mins))
else:
totalWrite = 0
readBytes += totalRead
writeBytes += totalWrite
readBytes = readBytes * 512
writeBytes = writeBytes * 512
peakReadBytes = peakReadBytes * 512
peakWriteBytes = peakWriteBytes * 512
if math.isnan(readBytes) or math.isnan(readBytes) or math.isnan(
readBytes) or math.isnan(readBytes):
print "findBlockIOusage, NAN found!!"
return readBytes, writeBytes, peakReadBytes, peakWriteBytes
def findIBUsage(execHostStr, jobStart, jobEnd):
global allIBRead
global allIBWrite
hostList = []
for host in execHostStr.split('+'):
hostList.append(host.split('/')[0])
hostList = list(set(hostList))
readBytes = 0.0
writeBytes = 0.0
peakReadBytes = 0.0
peakWriteBytes = 0.0
for host in hostList:
nodeNumber = int((host.split('-')[1])[1:])
readBytesPer10mins = taccExtract.extractTaccStats(taccDB,\
nodeNumber, 'ib_ext', 'port_rcv_data,E,U=4B',\
int(jobStart), int(jobEnd))
writeBytesPer10mins = taccExtract.extractTaccStats(taccDB,\
nodeNumber, 'ib_ext', 'port_xmit_data,E,U=4B',\
int(jobStart), int(jobEnd))
allIBWrite.extend(np.diff(readBytesPer10mins))
allIBRead.extend(np.diff(writeBytesPer10mins))
if len(readBytesPer10mins) > 0:
totalRead = readBytesPer10mins[-1] - readBytesPer10mins[0]
if len(readBytesPer10mins) > 1:
if peakReadBytes < np.amax(np.diff(readBytesPer10mins)):
peakReadBytes = np.amax(np.diff(readBytesPer10mins))
else:
totalRead = 0
if len(writeBytesPer10mins) > 0:
totalWrite = writeBytesPer10mins[-1] - writeBytesPer10mins[0]
if len(writeBytesPer10mins) > 1:
if peakWriteBytes < np.amax(np.diff(writeBytesPer10mins)):
peakWriteBytes = np.amax(np.diff(writeBytesPer10mins))
else:
totalWrite = 0
readBytes += totalRead
writeBytes += totalWrite
if math.isnan(readBytes) or math.isnan(readBytes) or math.isnan(
readBytes) or math.isnan(readBytes):
print "findIBEXTusage, NAN found!!"
return (readBytes * 4), (writeBytes * 4), (peakReadBytes *
4), (peakWriteBytes * 4)
def findPageFaults(execHostStr, jobStart, jobEnd):
global pgFltArray
hostList = []
for host in execHostStr.split('+'):
hostList.append(host.split('/')[0])
hostList = list(set(hostList))
minPgFlt = 0
maxPgFlt = 0
flag = False
for host in hostList:
nodeNumber = int((host.split('-')[1])[1:])
pagefaultsPer10Mins = taccExtract.extractTaccStats(
taccDB, nodeNumber, 'vm', 'pgmajfault,E', int(jobStart),
int(jobEnd))
# print host, nodeNumber, jobStart, int(jobStart), jobEnd, int(jobEnd)
# print pagefaultsPer10Mins
# print np.diff(pagefaultsPer10Mins)
# print np.amax(list(np.diff(pagefaultsPer10Mins)))
if len(pagefaultsPer10Mins) < 2:
maxPgFlt = 0
flag = True
elif maxPgFlt < np.amax(list(np.diff(pagefaultsPer10Mins))):
maxPgFlt = np.amax(list(np.diff(pagefaultsPer10Mins)))
pgFltArray.append(maxPgFlt)
return maxPgFlt, flag
def findMemUsage(execHostStr, jobStart, jobEnd):
hostList = []
for host in execHostStr.split('+'):
hostList.append(host.split('/')[0])
hostList = list(set(hostList))
peakMemUsed = []
avgMemUsed = []
for host in hostList:
nodeNumber = int((host.split('-')[1])[1:])
memChanges = taccExtract.extractTaccStats(taccDB,\
nodeNumber, 'mem', 'MemUsed,U=KB',\
int(jobStart), int(jobEnd))
if len(memChanges) > 0:
peakMemUsed.append(np.amax(memChanges))
avgMemUsed.append(np.mean(memChanges))
peakMem = np.mean(peakMemUsed)
avgMem = np.mean(avgMemUsed)
if math.isnan(peakMem):
peakMem = 0
if math.isnan(avgMem):
avgMem = 0
return peakMem, avgMem
def findPageFaults(execHostStr, jobStart, jobEnd):
global allPageFltDiffs
hostList = []
for host in execHostStr.split('+'):
hostList.append(host.split('/')[0])
hostList = list(set(hostList))
pageFlt = 0.0
peakPgFlt = 0.0
avgJumpsPgFlt = 0.0
flag = False
for host in hostList:
nodeNumber = int((host.split('-')[1])[1:])
pagefaultsPer10Mins = taccExtract.extractTaccStats(taccDB,\
nodeNumber, 'vm', 'pgmajfault,E',\
int(jobStart), int(jobEnd))
a = np.diff(pagefaultsPer10Mins)
allPageFltDiffs.extend(a)
# print a
# print pagefaultsPer10Mins, nodeNumber, jobStart, jobEnd
# raw_input()
if len(pagefaultsPer10Mins) > 0:
totalPageFaults = pagefaultsPer10Mins[-1] - pagefaultsPer10Mins[0]
if len(pagefaultsPer10Mins) > 1:
if peakPgFlt < np.amax(np.diff(pagefaultsPer10Mins)):
peakPgFlt = np.amax(np.diff(pagefaultsPer10Mins))
avgJumpsPgFlt = np.mean(np.diff(pagefaultsPer10Mins))
else:
totalPageFaults = 0
flag = True
if totalPageFaults > pageFlt:
pageFlt = totalPageFaults
return pageFlt, peakPgFlt, avgJumpsPgFlt, flag