class SimpleStageTester(object):
"""
A class for running simple tests of a Stage. Currently, this will run
only one Slice at a time, either a master slice (running the serial
preprocess() and postprocess() functions) or a worker slice (running
the process() function). The user is responsible for creating the
data that goes into the input clipboard.
"""
def __init__(self, stage=None, name="1", runID="simpleTest",
universeSize=1):
"""create the tester
@param stage a Stage instance for the first stage in a
simple pipeline.
@param name the name to associate with this stage (for
display purposes.
@param runID the run identifier to provide to the stage
@param universeSize the number of parallel threads to pretend are
running
"""
self.stage = stage
self.log = Debug("SimpleStageTester")
self.event = None
self.inQ = Queue()
self.outQ = Queue()
self.brokerhost = None
self.sysdata = {}
self.sysdata["runId"] = runID
self.sysdata["stageId"] = -1
self.sysdata["universeSize"] = universeSize
self.stages = []
if stage:
self.stages.append( (name, stage) )
def setEventBroker(self, host, port=None):
"""
set the event broker to use to get and receiver events. Pass in
None for the host to disable receiving events.
@param host the host where the event broker is running
@param port the port that the event broker is listening to
"""
self.brokerhost = host
if port:
self.brokerport = port
for stage in self.stages:
stage.setEventBroker(self.brokerhost)
def getEventBroker(self):
return (self.brokerhost, self.brokerport)
def addStage(self, stage, name=None, event=None):
"""
add a stage to this simple pipeline.
@param stage the Stage instance to add in order
@param name a name for the stage. If None, create one from the
stage object
@param event an event which must be received prior to execution
NOT YET IMPLEMENTED (ignored).
"""
if name is None:
name = str(len(self.stages))
stage.setEventBroker(self.brokerhost)
self.stages.append( (name, stage) )
def run(self, clipboard, sliceID):
"""run the stage as a particular slice, returning the output clipboard
@param clipboard a Clipboard or plain dictionary instance
containing the Stage input data
@param sliceID the number to give as the slice identifier. A
negative id number indicates the master slice.
"""
if (sliceID < 0):
return self.runMaster(clipboard, sliceID=sliceID)
else:
return self.runWorker(clipboard, sliceID)
def _makeStages(self, sliceID, isSerial):
sysdata = self.sysdata.copy()
stages = []
inQ = self.inQ
outQ = None
for name, stage in self.stages:
sysdata['stageId'] = len(stages) + 1
sysdata['name'] = name
if (isSerial):
newstage = stage.createSerialProcessing(self.log, sysdata)
else:
newstage = stage.createParallelProcessing(sliceID, self.log,
sysdata)
stages.append( newstage )
if len(stages) == len(self.stages):
outQ = self.outQ
else:
outQ = Queue()
stages[-1].initialize(outQ, inQ)
inQ = outQ
return stages
def runMaster(self, clipboard, sliceID=0):
"""run the Stage as a master, calling its initialize() function,
and then running the serial functions, preprocess() and postprocess()
"""
if isinstance(clipboard, dict):
clipboard = self.toClipboard(clipboard)
if not isinstance(clipboard, Clipboard):
raise TypeError("runMaster input is not a clipboard")
if self.event is not None:
clipboard(self.event[0], self.event[1])
stages = self._makeStages(sliceID, True)
self.inQ.addDataset(clipboard)
for stage in stages:
self.log.debug(5, "Calling Stage preprocess() on " +
stage.getName())
interQ = stage.applyPreprocess()
self.log.debug(5, "Stage preprocess() complete")
self.log.debug(5, "Calling Stage postprocess()")
stage.applyPostprocess(interQ)
self.log.debug(5, "Stage postprocess() complete")
return self.outQ.getNextDataset()
def runWorker(self, clipboard, sliceID=1):
"""run the Stage as a worker, running its process() function
@param clipboard a Clipboard or plain dictionary instance
containing the Stage input data
@param sliceID the number to give as the slice identifier;
must be >= 0.
"""
if isinstance(clipboard, dict):
clipboard = self.toClipboard(clipboard)
if not isinstance(clipboard, Clipboard):
raise TypeError("runMaster input is not a clipboard")
if self.event is not None:
clipboard(self.event[0], self.event[1])
stages = self._makeStages(sliceID, False)
self.inQ.addDataset(clipboard)
for stage in stages:
self.log.debug(5, "Calling Stage process()")
stage.applyProcess()
self.log.debug(5, "Stage process() complete")
return self.outQ.getNextDataset()
def setEvent(self, name, eventData):
"""set data for an event to be received by the Stage prior to
being called. This implementation currently does not support
array-valued data.
@param name the topic name for the event
@param eventData the event data, either as a python dictionary or
a PropertySet
"""
if isinstance(eventData, dict):
evprop = dafBase.PropertySet()
for key in eventData.keys():
evprop.set(key, eventData[key])
eventData = evprop
self.event = (name, eventData)
def toClipboard(self, data):
cb = Clipboard()
for key in data.keys():
cb.put(key, data[key])
return cb
def setDebugVerbosity(self, verbLimit):
"""set the verbosity of the default log. This and setLogThreshold()
are different APIs that affect the same underlying limit that controls
how many messages get logged.
@param verbLimit debug messages with a verbosity level larger
than this will not be printed. If positive
INFO, WARN, and FATAL messages will also
be printed.
"""
Log.getDefaultLog().setThreshold(-1*verbLimit)
def setLogThreshold(self, threshold):
"""set the importance threshold for the default log. This and
setDebugVerbosity are different APIs that affect the same underlying
limit that controls how many messages get logged. Normally one
uses one of the predefined values--Log.DEBUG, Log.INFO, Log.WARN, and
Log.FATAL--as input.
@param threshold the minimum importance of the message that will
get printed.
"""
Log.getDefaultLog().setThreshold(threshold)
def showAllLogProperties(self, show):
"""control whether log properties are displayed to the screen. These
include, for example, the DATE (and time) of the message.
@param show if true, show all the properties when a log message is
printed. If false, don't show them.
"""
Log.getDefaultLog().setShowAll(show)
def willShowAllLogProperties(self):
"""return whether log properties are displayed to the screen. These
include, for example, the DATE (and time) of the message.
"""
return Log.getDefaultLog().willShowAll()
class SimpleStageTester(object):
"""
A class for running simple tests of a Stage. Currently, this will run
only one Slice at a time, either a master slice (running the serial
preprocess() and postprocess() functions) or a worker slice (running
the process() function). The user is responsible for creating the
data that goes into the input clipboard.
"""
def __init__(self,
stage=None,
name="1",
runID="simpleTest",
universeSize=1):
"""create the tester
@param stage a Stage instance for the first stage in a
simple pipeline.
@param name the name to associate with this stage (for
display purposes.
@param runID the run identifier to provide to the stage
@param universeSize the number of parallel threads to pretend are
running
"""
self.stage = stage
self.log = Debug("SimpleStageTester")
self.event = None
self.inQ = Queue()
self.outQ = Queue()
self.brokerhost = None
self.sysdata = {}
self.sysdata["runId"] = runID
self.sysdata["stageId"] = -1
self.sysdata["universeSize"] = universeSize
self.stages = []
if stage:
self.stages.append((name, stage))
def setEventBroker(self, host, port=None):
"""
set the event broker to use to get and receiver events. Pass in
None for the host to disable receiving events.
@param host the host where the event broker is running
@param port the port that the event broker is listening to
"""
self.brokerhost = host
if port:
self.brokerport = port
for stage in self.stages:
stage.setEventBroker(self.brokerhost)
def getEventBroker(self):
return (self.brokerhost, self.brokerport)
def addStage(self, stage, name=None, event=None):
"""
add a stage to this simple pipeline.
@param stage the Stage instance to add in order
@param name a name for the stage. If None, create one from the
stage object
@param event an event which must be received prior to execution
NOT YET IMPLEMENTED (ignored).
"""
if name is None:
name = str(len(self.stages))
stage.setEventBroker(self.brokerhost)
self.stages.append((name, stage))
def run(self, clipboard, sliceID):
"""run the stage as a particular slice, returning the output clipboard
@param clipboard a Clipboard or plain dictionary instance
containing the Stage input data
@param sliceID the number to give as the slice identifier. A
negative id number indicates the master slice.
"""
if (sliceID < 0):
return self.runMaster(clipboard, sliceID=sliceID)
else:
return self.runWorker(clipboard, sliceID)
def _makeStages(self, sliceID, isSerial):
sysdata = self.sysdata.copy()
stages = []
inQ = self.inQ
outQ = None
for name, stage in self.stages:
sysdata['stageId'] = len(stages) + 1
sysdata['name'] = name
if (isSerial):
newstage = stage.createSerialProcessing(self.log, sysdata)
else:
newstage = stage.createParallelProcessing(
sliceID, self.log, sysdata)
stages.append(newstage)
if len(stages) == len(self.stages):
outQ = self.outQ
else:
outQ = Queue()
stages[-1].initialize(outQ, inQ)
inQ = outQ
return stages
def runMaster(self, clipboard, sliceID=0):
"""run the Stage as a master, calling its initialize() function,
and then running the serial functions, preprocess() and postprocess()
"""
if isinstance(clipboard, dict):
clipboard = self.toClipboard(clipboard)
if not isinstance(clipboard, Clipboard):
raise TypeError("runMaster input is not a clipboard")
if self.event is not None:
clipboard(self.event[0], self.event[1])
stages = self._makeStages(sliceID, True)
self.inQ.addDataset(clipboard)
for stage in stages:
self.log.debug(5,
"Calling Stage preprocess() on " + stage.getName())
interQ = stage.applyPreprocess()
self.log.debug(5, "Stage preprocess() complete")
self.log.debug(5, "Calling Stage postprocess()")
stage.applyPostprocess(interQ)
self.log.debug(5, "Stage postprocess() complete")
return self.outQ.getNextDataset()
def runWorker(self, clipboard, sliceID=1):
"""run the Stage as a worker, running its process() function
@param clipboard a Clipboard or plain dictionary instance
containing the Stage input data
@param sliceID the number to give as the slice identifier;
must be >= 0.
"""
if isinstance(clipboard, dict):
clipboard = self.toClipboard(clipboard)
if not isinstance(clipboard, Clipboard):
raise TypeError("runMaster input is not a clipboard")
if self.event is not None:
clipboard(self.event[0], self.event[1])
stages = self._makeStages(sliceID, False)
self.inQ.addDataset(clipboard)
for stage in stages:
self.log.debug(5, "Calling Stage process()")
stage.applyProcess()
self.log.debug(5, "Stage process() complete")
return self.outQ.getNextDataset()
def setEvent(self, name, eventData):
"""set data for an event to be received by the Stage prior to
being called. This implementation currently does not support
array-valued data.
@param name the topic name for the event
@param eventData the event data, either as a python dictionary or
a PropertySet
"""
if isinstance(eventData, dict):
evprop = dafBase.PropertySet()
for key in eventData.keys():
evprop.set(key, eventData[key])
eventData = evprop
self.event = (name, eventData)
def toClipboard(self, data):
cb = Clipboard()
for key in data.keys():
cb.put(key, data[key])
return cb
def setDebugVerbosity(self, verbLimit):
"""set the verbosity of the default log. This and setLogThreshold()
are different APIs that affect the same underlying limit that controls
how many messages get logged.
@param verbLimit debug messages with a verbosity level larger
than this will not be printed. If positive
INFO, WARN, and FATAL messages will also
be printed.
"""
Log.getDefaultLog().setThreshold(-1 * verbLimit)
def setLogThreshold(self, threshold):
"""set the importance threshold for the default log. This and
setDebugVerbosity are different APIs that affect the same underlying
limit that controls how many messages get logged. Normally one
uses one of the predefined values--Log.DEBUG, Log.INFO, Log.WARN, and
Log.FATAL--as input.
@param threshold the minimum importance of the message that will
get printed.
"""
Log.getDefaultLog().setThreshold(threshold)
def showAllLogProperties(self, show):
"""control whether log properties are displayed to the screen. These
include, for example, the DATE (and time) of the message.
@param show if true, show all the properties when a log message is
printed. If false, don't show them.
"""
Log.getDefaultLog().setShowAll(show)
def willShowAllLogProperties(self):
"""return whether log properties are displayed to the screen. These
include, for example, the DATE (and time) of the message.
"""
return Log.getDefaultLog().willShowAll()
