def plotOffsets(self, mpiFile, jobID, numProc):
offsets = mpiFile + "offsets" + jobID
offsetsFile = open(offsets, "rb")
while True:
file = offsetsFile.read(24)
if len(file) != 24:
break
else:
(id, beg, end) = struct.unpack("qqq", file)
if (id, beg, end) != (0.0, 0.0, 0.0):
self.axes.plot([id, id], [beg, end], color="b", \
linewidth=1.5)
offsetsFile.close()
(low, high) = self.axes.get_ybound()
self.axes.set_ylabel("Offset (Bytes x 1e%d)"% analysis.exponent(high))
self.axes.set_xlabel("Processor")
self.axes.set_xlim([-1, numProc])
def plotOffsets(self, mpiFile, jobID, numProc):
offsets = mpiFile + "offsets" + jobID
offsetsFile = open(offsets, "rb")
while True:
file = offsetsFile.read(24)
if len(file) != 24:
break
else:
(id, beg, end) = struct.unpack("qqq", file)
if (id, beg, end) != (0.0, 0.0, 0.0):
self.axes.plot([id, id], [beg, end], color="b", \
linewidth=1.5)
offsetsFile.close()
(low, high) = self.axes.get_ybound()
self.axes.set_ylabel("Offset (Bytes x 1e%d)" % analysis.exponent(high))
self.axes.set_xlabel("Processor")
self.axes.set_xlim([-1, numProc])
def graphPerProcessor(mpiFile,numberOfProcessors, average, jobID, above, below):
offsets = mpiFile + "offsets" + jobID
times = mpiFile + "times" + jobID
fig4 = plt.figure()
ax1 = fig4.add_subplot(2, 1, 1)
ax2 = fig4.add_subplot(2, 1, 2)
offsetFile = open(offsets, "rb")
title = ""
if above != 0:
title += "Number of Processors Ended Above Average: %d\n" % above
if below != 0:
title += "Number of Processors Ended Below Average: %d" % below
# 24 is the size of the number of bytes we are reading for 3 items
gc.disable()
inputList = []
while True:
file = offsetFile.read(24)
if len(file) != 24: # read the whole file
break
else:
(id, beg, end) = struct.unpack("qqq", file)
if (id, beg, end) != (0.0, 0.0, 0.0):
inputList.append((id, beg, end))
offsetFile.close();
gc.enable()
# so here we sort on id then by the beginning of each write in place
# this lets us downsample the output on the offset graph without losing
# *hopefully* too much real information at high write counts
inputList.sort( key = operator.itemgetter(0,1))
while len(inputList) > (100*numberOfProcessors) or len(inputList) > 65536:
inputList = inputList[::2] # get every other element
low = min(x[1] for x in inputList)
high = max(x[2] for x in inputList)
ax1.set_ylim([low,high])
ax1.set_xbound(lower=-1, upper= numberOfProcessors)
ax1.set_title(title, fontsize=10)
ax1.set_ylabel("Offset (Bytes x 1e%d)"% analysis.exponent(high))
ax1.set_autoscale_on(False)
#average
ax2.set_ylabel("Time")
ax2.set_xlim([-1, numberOfProcessors])
ax2.set_xlabel('Processor')
ax2.axhline(average, color="y", linestyle='dashed', linewidth=2)
ax2.set_autoscale_on(False)
#plot everything now that the axis are set
ax1.plot([[x[0] for x in inputList], [x[0] for x in inputList]],
[[x[1] for x in inputList], [x[2] for x in inputList]], color="b",
linewidth=1.5)
timeFile = open(times, "rb")
# 24 is the size of the number of bytes we are reading for 3 items
gc.disable()
inputList = []
while True:
file = timeFile.read(24)
if len(file) != 24:
break
else:
(id, beg, end) = struct.unpack("ddd", file)
if (id, beg, end) != (0.0, 0.0, 0.0):
inputList.append((id, beg, end))
gc.enable()
timeFile.close()
# Sometimes we have no times to parse if none are outside of one std dev.
if (len(inputList) > 0):
inputList.sort( key = operator.itemgetter(0,1))
while len(inputList) > (100*numberOfProcessors) or len(inputList) > 65536:
inputList = inputList[::2] # get every other element
low = min(x[1] for x in inputList)
high = max(x[2] for x in inputList)
ax2.set_ylim([low,high])
ax2.plot([[x[0] for x in inputList], [x[0] for x in inputList]],
[[x[1] for x in inputList], [x[2] for x in inputList]], linewidth=1.5)
def graphPerProcessor(offsets, timeProcessor, above, below, numStd, pids):
fig4 = plt.figure()
ax1 = fig4.add_subplot(2, 1, 1)
maxProcessor = 0
for item in offsets:
id = int(pids[item[0]])
beg = item[1]
end = item[2]
if maxProcessor < int(id):
maxProcessor = int(id)
ax1.plot([id, id], [beg, end], color="b", linewidth=1.5)
(low, high) = ax1.axes.get_ybound()
ax1.set_ylabel("Offset (Bytes x 1e%d)" % analysis.exponent(high))
# so the axis does not draw over processor 0
ax1.set_xlim([-1, maxProcessor + 1])
# list of ends so that we can find the cutoffs
ends = [0] * (maxProcessor + 1)
earliestStart = 0
ax2 = fig4.add_subplot(2, 1, 2)
for item in timeProcessor:
id = int(pids[item[0]])
currentEnd = ends[id]
beg = item[1]
end = item[2]
if earliestStart == 0:
earliestStart = float(beg)
if earliestStart > float(beg):
earliestStart = float(beg)
if currentEnd == 0:
ends[id] = float(end)
if currentEnd < float(end):
ends[id] = float(end)
for i in xrange(len(ends)):
ends[i] = ends[i] - earliestStart
endsArray = numpy.array(ends) # make a numpy array so it can calculate
# stdev and averages
avg = numpy.average(endsArray)
standardDev = numpy.std(endsArray)
aboveCutoff = avg + numStd * standardDev
belowCutoff = avg - numStd * standardDev
# get the number graphed above and below the cutoffs
numAbove = 0
numBelow = 0
for end in ends:
if maxProcessor <= 16: # all are graphed so count all of them
if end < avg:
numBelow += 1
if end > avg:
numAbove += 1
else: # these are those that are graphed with standard deviation cutoffs
if end < belowCutoff:
numBelow += 1
if end > aboveCutoff:
numAbove += 1
title = ""
if numAbove != 0:
title += "Number of Processors Ended Above Average: %d\n" % numAbove
if numBelow != 0:
title += "Number of Processors Ended Below Average: %d" % numBelow
ax1.set_title(title, fontsize=10)
for item in timeProcessor:
id = int(pids[item[0]])
beg = float(item[1]) - earliestStart
end = float(item[2]) - earliestStart
if maxProcessor <= 16: # graph all of the processors
ax2.plot([id, id], [beg, end], linewidth=1.5)
else: # graph based on inputs
if above and (ends[id] > aboveCutoff):
ax2.plot([id, id], [beg, end], linewidth=1.5)
if below and (ends[id] < belowCutoff):
ax2.plot([id, id], [beg, end], linewidth=1.5)
# average
ax2.plot(["0", str(maxProcessor)], [avg, avg], color="y", linewidth=2)
ax2.set_ylabel("Time")
ax2.set_xlim([-1, maxProcessor + 1])
ax2.set_xlabel("Processor")
def graphPerProcessor(mpiFile, numberOfProcessors, average, jobID, above,
below):
offsets = mpiFile + "offsets" + jobID
times = mpiFile + "times" + jobID
fig4 = plt.figure()
ax1 = fig4.add_subplot(2, 1, 1)
ax2 = fig4.add_subplot(2, 1, 2)
offsetFile = open(offsets, "rb")
title = ""
if above != 0:
title += "Number of Processors Ended Above Average: %d\n" % above
if below != 0:
title += "Number of Processors Ended Below Average: %d" % below
# 24 is the size of the number of bytes we are reading for 3 items
gc.disable()
inputList = []
while True:
file = offsetFile.read(24)
if len(file) != 24: # read the whole file
break
else:
(id, beg, end) = struct.unpack("qqq", file)
if (id, beg, end) != (0.0, 0.0, 0.0):
inputList.append((id, beg, end))
offsetFile.close()
gc.enable()
# so here we sort on id then by the beginning of each write in place
# this lets us downsample the output on the offset graph without losing
# *hopefully* too much real information at high write counts
inputList.sort(key=operator.itemgetter(0, 1))
while len(inputList) > (100 *
numberOfProcessors) or len(inputList) > 65536:
inputList = inputList[::2] # get every other element
low = min(x[1] for x in inputList)
high = max(x[2] for x in inputList)
ax1.set_ylim([low, high])
ax1.set_xbound(lower=-1, upper=numberOfProcessors)
ax1.set_title(title, fontsize=10)
ax1.set_ylabel("Offset (Bytes x 1e%d)" % analysis.exponent(high))
ax1.set_autoscale_on(False)
#average
ax2.set_ylabel("Time")
ax2.set_xlim([-1, numberOfProcessors])
ax2.set_xlabel('Processor')
ax2.axhline(average, color="y", linestyle='dashed', linewidth=2)
ax2.set_autoscale_on(False)
#plot everything now that the axis are set
ax1.plot([[x[0] for x in inputList], [x[0] for x in inputList]],
[[x[1] for x in inputList], [x[2] for x in inputList]],
color="b",
linewidth=1.5)
timeFile = open(times, "rb")
# 24 is the size of the number of bytes we are reading for 3 items
gc.disable()
inputList = []
while True:
file = timeFile.read(24)
if len(file) != 24:
break
else:
(id, beg, end) = struct.unpack("ddd", file)
if (id, beg, end) != (0.0, 0.0, 0.0):
inputList.append((id, beg, end))
gc.enable()
timeFile.close()
# Sometimes we have no times to parse if none are outside of one std dev.
if (len(inputList) > 0):
inputList.sort(key=operator.itemgetter(0, 1))
while len(inputList) > (100 *
numberOfProcessors) or len(inputList) > 65536:
inputList = inputList[::2] # get every other element
low = min(x[1] for x in inputList)
high = max(x[2] for x in inputList)
ax2.set_ylim([low, high])
ax2.plot([[x[0] for x in inputList], [x[0] for x in inputList]],
[[x[1] for x in inputList], [x[2] for x in inputList]],
linewidth=1.5)
def graphPerProcessor(offsets, timeProcessor, above, below, numStd, pids):
fig4 = plt.figure()
ax1 = fig4.add_subplot(2, 1, 1)
maxProcessor = 0
for item in offsets:
id = int(pids[item[0]])
beg = item[1]
end = item[2]
if maxProcessor < int(id):
maxProcessor = int(id)
ax1.plot([id, id], [beg, end], color="b", linewidth=1.5)
(low, high) = ax1.axes.get_ybound()
ax1.set_ylabel("Offset (Bytes x 1e%d)" % analysis.exponent(high))
#so the axis does not draw over processor 0
ax1.set_xlim([-1, maxProcessor + 1])
# list of ends so that we can find the cutoffs
ends = [0] * (maxProcessor + 1)
earliestStart = 0
ax2 = fig4.add_subplot(2, 1, 2)
for item in timeProcessor:
id = int(pids[item[0]])
currentEnd = ends[id]
beg = item[1]
end = item[2]
if earliestStart == 0:
earliestStart = float(beg)
if earliestStart > float(beg):
earliestStart = float(beg)
if currentEnd == 0:
ends[id] = float(end)
if currentEnd < float(end):
ends[id] = float(end)
for i in xrange(len(ends)):
ends[i] = ends[i] - earliestStart
endsArray = numpy.array(ends) #make a numpy array so it can calculate
#stdev and averages
avg = numpy.average(endsArray)
standardDev = numpy.std(endsArray)
aboveCutoff = avg + numStd * standardDev
belowCutoff = avg - numStd * standardDev
# get the number graphed above and below the cutoffs
numAbove = 0
numBelow = 0
for end in ends:
if maxProcessor <= 16: #all are graphed so count all of them
if end < avg:
numBelow += 1
if end > avg:
numAbove += 1
else: #these are those that are graphed with standard deviation cutoffs
if end < belowCutoff:
numBelow += 1
if end > aboveCutoff:
numAbove += 1
title = ""
if numAbove != 0:
title += "Number of Processors Ended Above Average: %d\n" % numAbove
if numBelow != 0:
title += "Number of Processors Ended Below Average: %d" % numBelow
ax1.set_title(title, fontsize=10)
for item in timeProcessor:
id = int(pids[item[0]])
beg = float(item[1]) - earliestStart
end = float(item[2]) - earliestStart
if maxProcessor <= 16: #graph all of the processors
ax2.plot([id, id], [beg, end], linewidth=1.5)
else: #graph based on inputs
if (above and (ends[id] > aboveCutoff)):
ax2.plot([id, id], [beg, end], linewidth=1.5)
if (below and (ends[id] < belowCutoff)):
ax2.plot([id, id], [beg, end], linewidth=1.5)
#average
ax2.plot(["0", str(maxProcessor)], [avg, avg], color="y", linewidth=2)
ax2.set_ylabel("Time")
ax2.set_xlim([-1, maxProcessor + 1])
ax2.set_xlabel('Processor')