def setUp(self):
self.imageRefPath = os.path.join('input', 'testImages')
self.resultsPath1 = os.path.join('input', 'resultsDirs', 'res1')
self.resultsPath2 = os.path.join('input', 'resultsDirs', 'res2')
# Set up a skeleton model run.
self.imageCompTest = ImageCompTC("window.00001.png", (0.1, 0.1),
refPath=self.imageRefPath)
self.resultsSet1 = ModelResult('testModel1', self.resultsPath1)
self.resultsSet2 = ModelResult('testModel2', self.resultsPath2)
def test_getAllResults(self):
mRes = ModelResult("test", './output')
fieldComps = FieldComparisonList()
fComp = FieldComparisonOp('TemperatureField')
fieldComps.add(fComp)
fResults = fieldComps.getAllResults(mRes)
self.assertEqual(len(fResults), 1)
self.assertEqual(fResults[0].fieldName, fComp.name)
self.assertEqual(fResults[0].dofErrors[0], 0.00612235812)
def createModelResult(self):
""" This is a new interface, to be called by JobRunner to generate
ModelResult. It returns a ModelResult object that matches the ModelRun
object.
Here it just use ModelResult (before refactoring)
"""
absOutPath = os.path.join(self.basePath, self.outputPath)
mres = ModelResult(self.name, absOutPath)
return mres
def plotAllRuns(outDir):
scalingDirs = [os.path.join(outDir, sub) for sub in os.walk(outDir).next()[1]]
scalingDirs.sort()
nDirs = len(scalingDirs)
cpuTimes = []
for sd in scalingDirs:
mRes = ModelResult(os.path.split(sd)[-1], sd)
mRes.readFrequentOutput()
fo = mRes.freqOutput
cpuTimes.append(plotCpuTimes.getPerStepCPU_Times(fo))
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(8, 4.0 + 1.0 * len(cpuTimes)))
fig.text(0.5, 0.95, "%s: CPU time/timestep" % outDir, horizontalalignment="center")
plt.subplots_adjust(hspace=0.4)
maxTSteps = max(map(len, cpuTimes))
subplots = []
for ii, cpuTimeList in enumerate(cpuTimes):
ax = plt.subplot(nDirs, 1, ii + 1)
subplots.append(ax)
plt.title(os.path.split(scalingDirs[ii])[-1])
plt.plot(cpuTimeList)
plt.ylim(0, math.ceil(max(cpuTimeList) * 10) / 10.0)
plt.xlim(0, maxTSteps)
majLocator = matplotlib.ticker.MaxNLocator(3)
ax.yaxis.set_major_locator(majLocator)
for ii in range(nDirs - 1):
plt.setp(subplots[ii].get_xticklabels(), visible=False)
subplots[-1].set_xlabel("timestep")
subplots[nDirs / 2].set_ylabel("Step Time (sec)")
plt.show()
plt.savefig(os.path.join(outDir, "cpuTimePerStep.png"))
def blockResult(self, modelRun, jobMI):
# CHeck jobMI is of type MPI ...
maxRunTime = modelRun.jobParams['maxRunTime']
pollInterval = modelRun.jobParams['pollInterval']
procHandle = jobMI.procHandle
# Navigate to the model's base directory
startDir = os.getcwd()
if modelRun.basePath != startDir:
print "Changing to ModelRun's specified base path '%s'" % \
(modelRun.basePath)
os.chdir(modelRun.basePath)
if maxRunTime == None or maxRunTime <= 0:
timeOut = False
retCode = procHandle.wait()
else:
if pollInterval > maxRunTime: pollInterval = maxRunTime
totalTime = 0
timeOut = True
while totalTime <= maxRunTime:
# Note: current strategy in this loop means 'totalTime'
# recorded here will only be as accurate as size of
# pollInterval.
# Thus this is a fall-back for recording time taken.
time.sleep(pollInterval)
totalTime += pollInterval
retCode = procHandle.poll()
if retCode is not None:
timeOut = False
break
if timeOut:
# At this point, we know the process has run too long.
# From Python 2.6, change this to procHandle.kill()
print "Error: passed timeout of %s, sending quit signal." % \
(str(timedelta(seconds=maxRunTime)))
os.kill(procHandle.pid, signal.SIGQUIT)
# TODO: set finishTime
# Check status of run (eg error status)
stdOutFilename = modelRun.getStdOutFilename()
stdErrFilename = modelRun.getStdErrFilename()
if timeOut == True:
raise ModelRunTimeoutError(modelRun.name, stdOutFilename,
stdErrFilename, maxRunTime)
if retCode != 0:
raise ModelRunRegularError(modelRun.name, retCode, stdOutFilename,
stdErrFilename)
else:
# Taking advantage of os.path.join functionality to automatically
# over-ride later absolute paths.
absOutPath = os.path.join(modelRun.basePath, modelRun.outputPath)
absLogPath = os.path.join(modelRun.basePath, modelRun.logPath)
print "Model ran successfully (output saved to path %s" %\
(absOutPath),
if absLogPath != absOutPath:
print ", std out & std error to %s" % (absLogPath),
print ")."
# Now tidy things up after the run.
jobMI.stdOutFile.close()
jobMI.stdErrFile.close()
print "Doing post-run tidyup:"
modelRun.postRunCleanup()
# Construct a modelResult
mResult = ModelResult(modelRun.name, absOutPath)
mResult.jobMetaInfo = jobMI
try:
#TODO: the below should be a standard method of ModelResult
tSteps, simTime = getSimInfoFromFreqOutput(mResult.outputPath)
except ValueError:
# For now, allow runs that didn't create a freq output
tSteps, simTime = None, None
#Now collect profiler performance info.
for profiler in self.profilers:
profiler.attachPerformanceInfo(jobMI, mResult)
if modelRun.basePath != startDir:
print "Restoring initial path '%s'" % \
(startDir)
os.chdir(startDir)
return mResult
def blockResult(self, modelRun, jobMetaInfo):
# Check jobMetaInfo is of type PBS
# via self.runType = "PBS"
startDir = os.getcwd()
if modelRun.basePath != startDir:
print "Changing to ModelRun's specified base path '%s'" % \
(modelRun.basePath)
os.chdir(modelRun.basePath)
jobID = jobMetaInfo.jobId
pollInterval = modelRun.jobParams['pollInterval']
checkOutput = 0
# NB: unlike with the MPI Job Runner, we don't check the "maxJobTime" here:- since that was encoded
# in the PBS Walltime used. Wait as long as necessary for job to be queued, run, and completed
# in PBS system.
pbsWaitTime = 0
gotResult = False
pbsError = False
while gotResult == False:
time.sleep(pollInterval)
pbsWaitTime += pollInterval
# check PBS job output ... (eg using qstat on jobID)
qstat = os.popen("qstat " + jobID).readlines()
qstatus = "%s" % (qstat)
# when the job has been submitted and we query the job ID we should get something like:
# if the job has ended:
#qstat: Unknown Job Id 3506.tweedle
# OR
#3505.tweedle cratonic30t2c3d2 WendySharples 00:15:16 E batch
# if the job has not commenced running or is still running:
#3505.tweedle cratonic30t2c3d2 WendySharples 00:15:16 Q batch
#3505.tweedle cratonic30t2c3d2 WendySharples 00:15:16 R batch
#3505.tweedle cratonic30t2c3d2 WendySharples 00:15:16 S batch
# if the job has not been able to be run:
#3505.tweedle cratonic30t2c3d2 WendySharples 00:15:16 C batch
# So if we break the command line up into an array of words separated by spaces:
qstatus = qstatus.split(" ")
#jobName and modelName MUST be THE SAME
for ii in range(len(qstatus)):
if qstatus[ii] == "Unknown":
print "job has already run\n"
gotResult = True
elif qstatus[ii] == "R":
print "job is still running\n"
elif qstatus[ii] == "Q":
print "job is queued\n"
elif qstatus[ii] == "C":
print "job is cancelled\n"
gotResult = True
pbsError = True
elif qstatus[ii] == "E":
print "job has ended\n"
gotResult = True
# Check status of run (eg error status)
# TODO: archive PBS file in modelRun output directory.
# TODO: connect/copy PBS stdout/error files to standard expected names.
stdOutFilename = modelRun.getStdOutFilename()
stdErrFilename = modelRun.getStdErrFilename()
#qdel = os.popen("qdel "+jobID).readlines()
if pbsError:
raise ModelRunRegularError(modelRun.name, -1, stdOutFilename,
stdErrFilename)
else:
absOutPath = os.path.join(modelRun.basePath, modelRun.outputPath)
absLogPath = os.path.join(modelRun.basePath, modelRun.logPath)
# TODO: Move and rename output and error files created by PBS,
# ... to stdOutFilename, stdErrFilename
# check PBS output file and make sure there's something in it
jobName = "%s" % (modelRun.name)
jobid = jobID.split(".")
jobNo = jobid[0]
fileName = jobName + ".o" + jobNo
f = open(fileName, 'r')
lines = f.read()
if lines == "":
print "error in file no output obtained\n"
raise ModelRunRegularError(modelRun.name, retCode,
stdOutFilename, stdErrFilename)
else:
print "Model ran successfully (output saved to %s, std out"\
" & std error to %s." % (absOutPath, absLogPath)
print "Doing post-run tidyup:"
modelRun.postRunCleanup()
# Construct a modelResult
mResult = ModelResult(modelRun.name, absOutPath)
# Now attach appropriate Job meta info
try:
tSteps, simTime = getSimInfoFromFreqOutput(modelRun.outputPath)
except ValueError:
# For now, allow runs that didn't create a freq output
tSteps, simTime = None, None
# Perhaps functions on jobMetaInfo?
# get provenance info
# attach provenance info
# get performance info
# attach performance info
mResult.jobMetaInfo = jobMetaInfo
# Navigate to the model's base directory
if modelRun.basePath != startDir:
print "Restoring initial path '%s'" % (startDir)
os.chdir(startDir)
return mResult
assert 'CPU_Time' in freqOutput.headers
cpuIncTimes = freqOutput.getValuesArray('CPU_Time')
cpuTimes = [0] * len(cpuIncTimes)
cpuTimes[0] = cpuIncTimes[0]
for ii, t in enumerate(cpuIncTimes[1:]):
cpuTimes[ii+1] = t - cpuIncTimes[ii]
return cpuTimes
def plot(cpuTimes, modelDir):
"""Given array of cpu times per timestep, and a model dir, draws an
on-screen plot."""
import matplotlib.pyplot as plt
plt.plot(cpuTimes)
plt.xlabel("timestep")
plt.ylim(0, math.ceil(max(cpuTimes)*10)/10.0)
plt.ylabel("Time (sec)")
plt.title("CPU Time per timestep\n(%s)" % modelDir)
plt.show()
if __name__ == "__main__":
try:
outputPath = sys.argv[1]
except:
print "Bad input arguments - pass output path to model result."
raise
mRes = ModelResult("plotModel", outputPath)
mRes.readFrequentOutput()
fo = mRes.freqOutput
cpuTimes = getPerStepCPU_Times(fo)
plot(cpuTimes, outputPath)
def blockResult(self, modelRun, jobMI):
# CHeck jobMI is of type MPI ...
maxRunTime = modelRun.jobParams['maxRunTime']
pollInterval = modelRun.jobParams['pollInterval']
procHandle = jobMI.procHandle
# Navigate to the model's base directory
startDir = os.getcwd()
if modelRun.basePath != startDir:
print "Changing to ModelRun's specified base path '%s'" % \
(modelRun.basePath)
os.chdir(modelRun.basePath)
if maxRunTime == None or maxRunTime <= 0:
timeOut = False
retCode = procHandle.wait()
else:
if pollInterval > maxRunTime: pollInterval = maxRunTime
totalTime = 0
timeOut = True
while totalTime <= maxRunTime:
# Note: current strategy in this loop means 'totalTime'
# recorded here will only be as accurate as size of
# pollInterval.
# Thus this is a fall-back for recording time taken.
time.sleep(pollInterval)
totalTime += pollInterval
retCode = procHandle.poll()
if retCode is not None:
timeOut = False
break
if timeOut:
# At this point, we know the process has run too long.
# From Python 2.6, change this to procHandle.kill()
print "Error: passed timeout of %s, sending quit signal." % \
(str(timedelta(seconds=maxRunTime)))
os.kill(procHandle.pid, signal.SIGQUIT)
# TODO: set finishTime
# Check status of run (eg error status)
stdOutFilename = modelRun.getStdOutFilename()
stdErrFilename = modelRun.getStdErrFilename()
if timeOut == True:
raise ModelRunTimeoutError(modelRun.name, stdOutFilename,
stdErrFilename, maxRunTime)
if retCode != 0:
raise ModelRunRegularError(modelRun.name, retCode, stdOutFilename,
stdErrFilename)
else:
# Taking advantage of os.path.join functionality to automatically
# over-ride later absolute paths.
absOutPath = os.path.join(modelRun.basePath, modelRun.outputPath)
absLogPath = os.path.join(modelRun.basePath, modelRun.logPath)
print "Model ran successfully (output saved to path %s" %\
(absOutPath),
if absLogPath != absOutPath:
print ", std out & std error to %s" % (absLogPath),
print ")."
# Now tidy things up after the run.
jobMI.stdOutFile.close()
jobMI.stdErrFile.close()
print "Doing post-run tidyup:"
modelRun.postRunCleanup()
# Construct a modelResult
mResult = ModelResult(modelRun.name, absOutPath)
mResult.jobMetaInfo = jobMI
try:
#TODO: the below should be a standard method of ModelResult
tSteps, simTime = getSimInfoFromFreqOutput(mResult.outputPath)
except ValueError:
# For now, allow runs that didn't create a freq output
tSteps, simTime = None, None
#Now collect profiler performance info.
for profiler in self.profilers:
profiler.attachPerformanceInfo(jobMI, mResult)
if modelRun.basePath != startDir:
print "Restoring initial path '%s'" % \
(startDir)
os.chdir(startDir)
return mResult
def blockResult(self, modelRun, jobMetaInfo):
# Check jobMetaInfo is of type PBS
# via self.runType = "PBS"
startDir = os.getcwd()
if modelRun.basePath != startDir:
print "Changing to ModelRun's specified base path '%s'" % \
(modelRun.basePath)
os.chdir(modelRun.basePath)
jobID = jobMetaInfo.jobId
pollInterval = modelRun.jobParams['pollInterval']
checkOutput = 0
# NB: unlike with the MPI Job Runner, we don't check the "maxJobTime" here:- since that was encoded
# in the PBS Walltime used. Wait as long as necessary for job to be queued, run, and completed
# in PBS system.
pbsWaitTime = 0
gotResult = False
pbsError = False
while gotResult == False:
time.sleep(pollInterval)
pbsWaitTime += pollInterval
# check PBS job output ... (eg using qstat on jobID)
qstat = os.popen("qstat "+jobID).readlines()
qstatus = "%s" % (qstat)
# when the job has been submitted and we query the job ID we should get something like:
# if the job has ended:
#qstat: Unknown Job Id 3506.tweedle
# OR
#3505.tweedle cratonic30t2c3d2 WendySharples 00:15:16 E batch
# if the job has not commenced running or is still running:
#3505.tweedle cratonic30t2c3d2 WendySharples 00:15:16 Q batch
#3505.tweedle cratonic30t2c3d2 WendySharples 00:15:16 R batch
#3505.tweedle cratonic30t2c3d2 WendySharples 00:15:16 S batch
# if the job has not been able to be run:
#3505.tweedle cratonic30t2c3d2 WendySharples 00:15:16 C batch
# So if we break the command line up into an array of words separated by spaces:
qstatus = qstatus.split(" ")
#jobName and modelName MUST be THE SAME
for ii in range(len(qstatus)):
if qstatus[ii] == "Unknown":
print "job has already run\n"
gotResult = True
elif qstatus[ii] == "R":
print "job is still running\n"
elif qstatus[ii] == "Q":
print "job is queued\n"
elif qstatus[ii] == "C":
print "job is cancelled\n"
gotResult = True
pbsError = True
elif qstatus[ii] == "E":
print "job has ended\n"
gotResult = True
# Check status of run (eg error status)
# TODO: archive PBS file in modelRun output directory.
# TODO: connect/copy PBS stdout/error files to standard expected names.
stdOutFilename = modelRun.getStdOutFilename()
stdErrFilename = modelRun.getStdErrFilename()
#qdel = os.popen("qdel "+jobID).readlines()
if pbsError:
raise ModelRunRegularError(modelRun.name, -1, stdOutFilename,
stdErrFilename)
else:
absOutPath = os.path.join(modelRun.basePath, modelRun.outputPath)
absLogPath = os.path.join(modelRun.basePath, modelRun.logPath)
# TODO: Move and rename output and error files created by PBS,
# ... to stdOutFilename, stdErrFilename
# check PBS output file and make sure there's something in it
jobName = "%s" % (modelRun.name)
jobid = jobID.split(".")
jobNo = jobid[0]
fileName = jobName+".o"+jobNo
f = open(fileName, 'r')
lines = f.read()
if lines == "":
print "error in file no output obtained\n"
raise ModelRunRegularError(modelRun.name, retCode,
stdOutFilename, stdErrFilename)
else:
print "Model ran successfully (output saved to %s, std out"\
" & std error to %s." % (absOutPath, absLogPath)
print "Doing post-run tidyup:"
modelRun.postRunCleanup()
# Construct a modelResult
mResult = ModelResult(modelRun.name, absOutPath)
# Now attach appropriate Job meta info
try:
tSteps, simTime = getSimInfoFromFreqOutput(modelRun.outputPath)
except ValueError:
# For now, allow runs that didn't create a freq output
tSteps, simTime = None, None
# Perhaps functions on jobMetaInfo?
# get provenance info
# attach provenance info
# get performance info
# attach performance info
mResult.jobMetaInfo = jobMetaInfo
# Navigate to the model's base directory
if modelRun.basePath != startDir:
print "Restoring initial path '%s'" % (startDir)
os.chdir(startDir)
return mResult
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## Lesser General Public License for more details.
##
## You should have received a copy of the GNU Lesser General Public
## License along with this library; if not, write to the Free Software
## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
## MA 02110-1301 USA
from credo.modelresult import ModelResult
from credo.analysis.fields import FieldComparisonOp, FieldComparisonList
from credo.io import stgcvg
fComps = FieldComparisonList()
fComps.add(FieldComparisonOp('VelocityField'))
# TODO: perhaps should be an interface that doesn't require a full mRes?
mRes = ModelResult("testMod", "./output/realistic")
results = fComps.getAllResults(mRes)
fr = results[0]
fr.plotOverTime(show=True, dofIndex=0, path="./output/realistic")
#Plotting
#dofErrors = stgcvg.getDofErrors_ByDof( fr.cvgFileInfo )
#import matplotlib
#matplotlib.use('Agg')
#import matplotlib.pyplot as plt
#plt.plot(dofErrors[0])
#plt.axhline(y=fr.tol, label='tolerance', linewidth=3, color='r')
#plt.xlabel("Timestep")
cpuTimes = [0] * len(cpuIncTimes)
cpuTimes[0] = cpuIncTimes[0]
for ii, t in enumerate(cpuIncTimes[1:]):
cpuTimes[ii + 1] = t - cpuIncTimes[ii]
return cpuTimes
def plot(cpuTimes, modelDir):
"""Given array of cpu times per timestep, and a model dir, draws an
on-screen plot."""
import matplotlib.pyplot as plt
plt.plot(cpuTimes)
plt.xlabel("timestep")
plt.ylim(0, math.ceil(max(cpuTimes) * 10) / 10.0)
plt.ylabel("Time (sec)")
plt.title("CPU Time per timestep\n(%s)" % modelDir)
plt.show()
if __name__ == "__main__":
try:
outputPath = sys.argv[1]
except:
print "Bad input arguments - pass output path to model result."
raise
mRes = ModelResult("plotModel", outputPath)
mRes.readFrequentOutput()
fo = mRes.freqOutput
cpuTimes = getPerStepCPU_Times(fo)
plot(cpuTimes, outputPath)
def test_getResult(self):
mRes = ModelResult("test", './output/')
fComp = FieldComparisonOp('TemperatureField')
fRes = fComp.getResult(mRes)
self.assertEqual(fRes.fieldName, fComp.name)
self.assertEqual(fRes.dofErrors[0], 0.00612235812)
