def startStagesLoop(self):
"""
Execute the Stage loop. The loop progressing in step with
the analogous stage loop in the central Pipeline by means of
MPI Bcast and Barrier calls.
"""
startStagesLoopLog = self.log.timeBlock("startStagesLoop", self.TRACE)
looplog = BlockTimingLog(self.log, "visit", self.TRACE)
stagelog = BlockTimingLog(looplog, "stage", self.TRACE)
self.log.log(Log.INFO, "Begin startStagesLoopLog")
self.threadBarrier()
visitcount = 0
while True:
self.log.log(Log.INFO, "visitcount %d %s " % (visitcount, datetime.datetime.now()))
if ((self.executionMode == 1) and (visitcount == 1)):
LogRec(looplog, Log.INFO) << "terminating Slice Stage Loop "
# self.cppPipeline.invokeShutdown()
break
visitcount += 1
looplog.setPreamblePropertyInt("LOOPNUM", visitcount)
stagelog.setPreamblePropertyInt("LOOPNUM", visitcount)
# stagelog.setPreamblePropertyInt("stagename", visitcount)
timesVisitStart = os.times()
# looplog.setPreamblePropertyFloat("usertime", timesVisitStart[0])
# looplog.setPreamblePropertyFloat("systemtime", timesVisitStart[1])
looplog.setPreamblePropertyDouble("usertime", timesVisitStart[0])
looplog.setPreamblePropertyDouble("systemtime", timesVisitStart[1])
looplog.start()
self.startInitQueue() # place an empty clipboard in the first Queue
self.errorFlagged = 0
for iStage in range(1, self.nStages+1):
stagelog.setPreamblePropertyInt("STAGEID", iStage)
stagelog.setPreamblePropertyString("stagename", self.stageNames[iStage-1])
stagelog.start(self.stageNames[iStage-1] + " loop")
stagelog.log(Log.INFO, "Begin stage loop iteration iStage %d " % iStage)
stageObject = self.stageList[iStage-1]
self.handleEvents(iStage, stagelog)
# synchronize before preprocess
self.threadBarrier()
# synchronize after preprocess, before process
self.threadBarrier()
self.tryProcess(iStage, stageObject, stagelog)
# synchronize after process, before postprocess
self.threadBarrier()
# synchronize after postprocess
self.threadBarrier()
stagelog.log(self.TRACE, "End stage loop iteration iStage %d " % iStage)
stagelog.log(Log.INFO, "End stage loop iteration : ErrorCheck \
iStage %d stageName %s errorFlagged_%d " % (iStage, self.stageNames[iStage-1], self.errorFlagged) )
stagelog.done()
looplog.log(self.VERB2, "Completed Stage Loop")
# If no error/exception was flagged, then clear the final Clipboard in the final Queue
if self.errorFlagged == 0:
looplog.log(Log.DEBUG,
"Retrieving final Clipboard for deletion")
finalQueue = self.queueList[self.nStages]
finalClipboard = finalQueue.getNextDataset()
finalClipboard.close()
del finalClipboard
looplog.log(Log.DEBUG, "Deleted final Clipboard")
else:
looplog.log(self.VERB3, "Error flagged on this visit")
timesVisitDone = os.times()
utime = timesVisitDone[0] - timesVisitStart[0]
stime = timesVisitDone[1] - timesVisitStart[1]
wtime = timesVisitDone[4] - timesVisitStart[4]
totalTime = utime + stime
looplog.log(Log.INFO, "visittimes : utime %.4f stime %.4f total %.4f wtime %.4f" % (utime, stime, totalTime, wtime) )
# looplog.setPreamblePropertyFloat("usertime", timesVisitDone[0])
# looplog.setPreamblePropertyFloat("systemtime", timesVisitDone[1])
looplog.setPreamblePropertyDouble("usertime", timesVisitDone[0])
looplog.setPreamblePropertyDouble("systemtime", timesVisitDone[1])
looplog.done()
try:
memmsg = "mem:"
with open("/proc/%d/status" % os.getpid(), "r") as f:
for l in f:
m = re.match(r'Vm(Size|RSS|Peak|HWM):\s+(\d+ \wB)', l)
if m:
memmsg += " %s=%s" % m.groups()
looplog.log(Log.INFO, memmsg)
except:
pass
# LogRec(looplog, Log.INFO) << Prop("usertime", utime) \
# << Prop("systemtime", stime) \
# << LogRec.endr;
startStagesLoopLog.done()
q3 = lsst.pex.harness.Queue.Queue()
q4 = lsst.pex.harness.Queue.Queue()
sysdata = {}
sysdata["name"] = "testPipeline"
sysdata["rank"] = 0
sysdata["stageId"] = 1
sysdata["universeSize"] = 100
sysdata["runId"] = "testrun"
eventBrokerHost = "lsst4.ncsa.uiuc.edu"
ScreenLog.createDefaultLog(True)
log = BlockTimingLog(Log.getDefaultLog(), "TestIO", BlockTimingLog.INSTRUM)
log.setUsageFlags(log.ALLUDATA)
log.setThreshold(log.DEBUG)
log.log(log.INFO, "hello")
# Create and initialize the stages
outputStage = lsst.pex.harness.IOStage.OutputStageParallel(outputPolicy, log, eventBrokerHost, sysdata)
outputStage.initialize(q2, q1)
sysdata["rank"] = -1
inputStage = lsst.pex.harness.IOStage.InputStageSerial(inputPolicy, log, eventBrokerHost, sysdata)
inputStage.initialize(q4, q3)
# Note: no direct connection between the stages!
# Create the event PropertySet
event = lsst.daf.base.PropertySet()
event.addString("visitId", "fov391")
# Create the clipboard and put the event on it
def startStagesLoop(self):
"""
Method to execute loop over Stages
"""
startStagesLoopLog = self.log.timeBlock("startStagesLoop", self.TRACE)
looplog = BlockTimingLog(self.log, "visit", self.TRACE)
stagelog = BlockTimingLog(looplog, "stage", self.TRACE-1)
proclog = BlockTimingLog(stagelog, "process", self.TRACE)
visitcount = 0
self.threadBarrier(0)
while True:
if (((self.executionMode == 1) and (visitcount == 1))):
LogRec(looplog, Log.INFO) << "terminating pipeline after one loop/visit "
#
# Need to shutdown Threads here
#
break
else:
visitcount += 1
looplog.setPreamblePropertyInt("LOOPNUM", visitcount)
looplog.start()
stagelog.setPreamblePropertyInt("LOOPNUM", visitcount)
proclog.setPreamblePropertyInt("LOOPNUM", visitcount)
# self.cppPipeline.invokeContinue()
self.startInitQueue() # place an empty clipboard in the first Queue
self.errorFlagged = 0
for iStage in range(1, self.nStages+1):
stagelog.setPreamblePropertyInt("STAGEID", iStage)
stagelog.start(self.stageNames[iStage-1] + " loop")
proclog.setPreamblePropertyInt("STAGEID", iStage)
stage = self.stageList[iStage-1]
self.handleEvents(iStage, stagelog)
# synchronize before preprocess
self.threadBarrier(iStage)
self.tryPreProcess(iStage, stage, stagelog)
# synchronize after preprocess, before process
self.threadBarrier(iStage)
# synchronize after process, before postprocess
self.threadBarrier(iStage)
self.tryPostProcess(iStage, stage, stagelog)
# synchronize after postprocess
self.threadBarrier(iStage)
stagelog.done()
self.checkExitByStage()
else:
looplog.log(self.VERB2, "Completed Stage Loop")
self.checkExitByVisit()
# Uncomment to print a list of Citizens after each visit
# print datap.Citizen_census(0,0), "Objects:"
# print datap.Citizen_census(datap.cout,0)
looplog.log(Log.DEBUG, 'Retrieving finalClipboard for deletion')
finalQueue = self.queueList[self.nStages]
finalClipboard = finalQueue.getNextDataset()
looplog.log(Log.DEBUG, "deleting final clipboard")
looplog.done()
# delete entries on the clipboard
finalClipboard.close()
del finalClipboard
startStagesLoopLog.log(Log.INFO, "Shutting down pipeline");
self.shutdown()
startStagesLoopLog.done()
q3 = lsst.pex.harness.Queue.Queue()
q4 = lsst.pex.harness.Queue.Queue()
sysdata = {}
sysdata["name"] = "testPipeline"
sysdata["rank"] = 0
sysdata["stageId"] = 1
sysdata["universeSize"] = 100
sysdata["runId"] = "testrun"
eventBrokerHost = "lsst4.ncsa.uiuc.edu"
ScreenLog.createDefaultLog(True)
log = BlockTimingLog(Log.getDefaultLog(), "TestIO", BlockTimingLog.INSTRUM)
log.setUsageFlags(log.ALLUDATA)
log.setThreshold(log.DEBUG)
log.log(log.INFO, "hello")
# Create and initialize the stages
outputStage = lsst.pex.harness.IOStage.OutputStageParallel(
outputPolicy, log, eventBrokerHost, sysdata)
outputStage.initialize(q2, q1)
sysdata["rank"] = -1
inputStage = lsst.pex.harness.IOStage.InputStageSerial(inputPolicy, log,
eventBrokerHost,
sysdata)
inputStage.initialize(q4, q3)
# Note: no direct connection between the stages!
# Create the event PropertySet
event = lsst.daf.base.PropertySet()
def startStagesLoop(self):
"""
Execute the Stage loop. The loop progressing in step with
the analogous stage loop in the central Pipeline by means of
MPI Bcast and Barrier calls.
"""
startStagesLoopLog = self.log.timeBlock("startStagesLoop", self.TRACE)
looplog = BlockTimingLog(self.log, "visit", self.TRACE)
stagelog = BlockTimingLog(looplog, "stage", self.TRACE)
self.log.log(Log.INFO, "Begin startStagesLoopLog")
self.threadBarrier()
visitcount = 0
while True:
self.log.log(
Log.INFO,
"visitcount %d %s " % (visitcount, datetime.datetime.now()))
if ((self.executionMode == 1) and (visitcount == 1)):
LogRec(looplog, Log.INFO) << "terminating Slice Stage Loop "
# self.cppPipeline.invokeShutdown()
break
visitcount += 1
looplog.setPreamblePropertyInt("LOOPNUM", visitcount)
stagelog.setPreamblePropertyInt("LOOPNUM", visitcount)
# stagelog.setPreamblePropertyInt("stagename", visitcount)
timesVisitStart = os.times()
# looplog.setPreamblePropertyFloat("usertime", timesVisitStart[0])
# looplog.setPreamblePropertyFloat("systemtime", timesVisitStart[1])
looplog.setPreamblePropertyDouble("usertime", timesVisitStart[0])
looplog.setPreamblePropertyDouble("systemtime", timesVisitStart[1])
looplog.start()
self.startInitQueue(
) # place an empty clipboard in the first Queue
self.errorFlagged = 0
for iStage in range(1, self.nStages + 1):
stagelog.setPreamblePropertyInt("STAGEID", iStage)
stagelog.setPreamblePropertyString("stagename",
self.stageNames[iStage - 1])
stagelog.start(self.stageNames[iStage - 1] + " loop")
stagelog.log(Log.INFO,
"Begin stage loop iteration iStage %d " % iStage)
stageObject = self.stageList[iStage - 1]
self.handleEvents(iStage, stagelog)
# synchronize before preprocess
self.threadBarrier()
# synchronize after preprocess, before process
self.threadBarrier()
self.tryProcess(iStage, stageObject, stagelog)
# synchronize after process, before postprocess
self.threadBarrier()
# synchronize after postprocess
self.threadBarrier()
stagelog.log(self.TRACE,
"End stage loop iteration iStage %d " % iStage)
stagelog.log(
Log.INFO, "End stage loop iteration : ErrorCheck \
iStage %d stageName %s errorFlagged_%d " %
(iStage, self.stageNames[iStage - 1], self.errorFlagged))
stagelog.done()
looplog.log(self.VERB2, "Completed Stage Loop")
# If no error/exception was flagged, then clear the final Clipboard in the final Queue
if self.errorFlagged == 0:
looplog.log(Log.DEBUG,
"Retrieving final Clipboard for deletion")
finalQueue = self.queueList[self.nStages]
finalClipboard = finalQueue.getNextDataset()
finalClipboard.close()
del finalClipboard
looplog.log(Log.DEBUG, "Deleted final Clipboard")
else:
looplog.log(self.VERB3, "Error flagged on this visit")
timesVisitDone = os.times()
utime = timesVisitDone[0] - timesVisitStart[0]
stime = timesVisitDone[1] - timesVisitStart[1]
wtime = timesVisitDone[4] - timesVisitStart[4]
totalTime = utime + stime
looplog.log(
Log.INFO,
"visittimes : utime %.4f stime %.4f total %.4f wtime %.4f" %
(utime, stime, totalTime, wtime))
# looplog.setPreamblePropertyFloat("usertime", timesVisitDone[0])
# looplog.setPreamblePropertyFloat("systemtime", timesVisitDone[1])
looplog.setPreamblePropertyDouble("usertime", timesVisitDone[0])
looplog.setPreamblePropertyDouble("systemtime", timesVisitDone[1])
looplog.done()
try:
memmsg = "mem:"
with open("/proc/%d/status" % os.getpid(), "r") as f:
for l in f:
m = re.match(r'Vm(Size|RSS|Peak|HWM):\s+(\d+ \wB)', l)
if m:
memmsg += " %s=%s" % m.groups()
looplog.log(Log.INFO, memmsg)
except:
pass
# LogRec(looplog, Log.INFO) << Prop("usertime", utime) \
# << Prop("systemtime", stime) \
# << LogRec.endr;
startStagesLoopLog.done()
def startStagesLoop(self):
"""
Method to execute loop over Stages
"""
startStagesLoopLog = self.log.timeBlock("startStagesLoop", self.TRACE)
looplog = BlockTimingLog(self.log, "visit", self.TRACE)
stagelog = BlockTimingLog(looplog, "stage", self.TRACE - 1)
proclog = BlockTimingLog(stagelog, "process", self.TRACE)
visitcount = 0
self.threadBarrier(0)
while True:
if (((self.executionMode == 1) and (visitcount == 1))):
LogRec(
looplog,
Log.INFO) << "terminating pipeline after one loop/visit "
#
# Need to shutdown Threads here
#
break
else:
visitcount += 1
looplog.setPreamblePropertyInt("LOOPNUM", visitcount)
looplog.start()
stagelog.setPreamblePropertyInt("LOOPNUM", visitcount)
proclog.setPreamblePropertyInt("LOOPNUM", visitcount)
# self.cppPipeline.invokeContinue()
self.startInitQueue(
) # place an empty clipboard in the first Queue
self.errorFlagged = 0
for iStage in range(1, self.nStages + 1):
stagelog.setPreamblePropertyInt("STAGEID", iStage)
stagelog.start(self.stageNames[iStage - 1] + " loop")
proclog.setPreamblePropertyInt("STAGEID", iStage)
stage = self.stageList[iStage - 1]
self.handleEvents(iStage, stagelog)
# synchronize before preprocess
self.threadBarrier(iStage)
self.tryPreProcess(iStage, stage, stagelog)
# synchronize after preprocess, before process
self.threadBarrier(iStage)
# synchronize after process, before postprocess
self.threadBarrier(iStage)
self.tryPostProcess(iStage, stage, stagelog)
# synchronize after postprocess
self.threadBarrier(iStage)
stagelog.done()
self.checkExitByStage()
else:
looplog.log(self.VERB2, "Completed Stage Loop")
self.checkExitByVisit()
# Uncomment to print a list of Citizens after each visit
# print datap.Citizen_census(0,0), "Objects:"
# print datap.Citizen_census(datap.cout,0)
looplog.log(Log.DEBUG, 'Retrieving finalClipboard for deletion')
finalQueue = self.queueList[self.nStages]
finalClipboard = finalQueue.getNextDataset()
looplog.log(Log.DEBUG, "deleting final clipboard")
looplog.done()
# delete entries on the clipboard
finalClipboard.close()
del finalClipboard
startStagesLoopLog.log(Log.INFO, "Shutting down pipeline")
self.shutdown()
startStagesLoopLog.done()