def __init__(self, _strPluginName):
"""
Constructor of the class
@param strPluginName: name of the plugin
@type strPluginName: string
"""
EDLogging.__init__(self)
self.__strPluginName = _strPluginName
self.__edPlugin = None
self.__edSlotCallBack = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__pathXSDInput = None
self.__pathXSDOutput = None
self.__bXmlInputSet = False
self.__status = None
self.__name = None
self.__runtime = None
self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName)
if self.__edPlugin is None:
raise RuntimeError("Unable to create plugin %s" % self.__strPluginName)
self.__jobId = "%s-%08i" % (self.__strPluginName, self.__edPlugin.getId())
with self.__class__.__semaphore:
self.__class__.__dictJobs[self.__jobId] = self
if (self.__edPlugin is None):
self.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName)
else:
self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def __init__(self, _strPluginName):
"""
Constructor of the class
@param strPluginName: name of the plugin
@type strPluginName: string
"""
EDObject.__init__(self)
self.__strPluginName = _strPluginName
self.__edPlugin = None
self.__edSlotCallBack = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__bXmlInputSet = False
self.__status = None
# self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName)
self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName)
EDJob.__semaphore.acquire()
# Create the JobID
if not self.__strPluginName in EDJob.__dictPluginLastId:
EDJob.__dictPluginLastId[_strPluginName] = 0
else:
EDJob.__dictPluginLastId[_strPluginName] += 1
self.__jobId = "%s-%i" % (self.__strPluginName, EDJob.__dictPluginLastId[_strPluginName])
EDJob.__dictJobs[self.__jobId] = self
EDJob.__semaphore.release()
if (self.__edPlugin is None):
EDVerbose.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName)
else:
self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def __init__ (self):
"""
Initializes plugin related attributes described above
"""
EDAction.__init__(self)
self.__xsPluginItem = None
self.__dictXSDataInputClass = {}
self.__dictXSDataInput = {}
self.__dictXSDataOutput = {}
self.__strDefaultInputDataKey = "defaultInputData"
self.__strDefaultOutputDataKey = "defaultOutputData"
self.__edSlotExportDataOutput = EDSlot()
self.__strBaseDirectory = None
self.__strWorkingDirectory = None
self.__strBaseName = None
self.__listExecutiveSummaryLines = []
self.__strExecutiveSummarySeparator = "-" * 80
self.__listErrorMessages = []
self.__listWarningMessages = []
self.__isRequiredToHaveConfiguration = False
self.__bWriteDataXMLInputOutput = True
self.__bWriteDataXMLOutput = True
self.__bWriteDataXMLInput = True
self.__strPluginId = "%s-%08i" % (self.getClassName(), self.getId())
self.strPathDataInput = None
self.strPathDataOutput = None
def __init__(self, _strPluginName):
"""
Constructor of the class
@param strPluginName: name of the plugin
@type strPluginName: string
"""
EDObject.__init__(self)
self.__strPluginName = _strPluginName
self.__edPlugin = None
self.__edSlotCallBack = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__bXmlInputSet = False
self.__status = None
# self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName)
self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName)
EDJob.__semaphore.acquire()
# Create the JobID
if not self.__strPluginName in EDJob.__dictPluginLastId:
EDJob.__dictPluginLastId[_strPluginName] = 0
else:
EDJob.__dictPluginLastId[_strPluginName] += 1
self.__jobId = "%s-%i" % (self.__strPluginName,
EDJob.__dictPluginLastId[_strPluginName])
EDJob.__dictJobs[self.__jobId] = self
EDJob.__semaphore.release()
if (self.__edPlugin is None):
EDVerbose.WARNING("Instantiation of plugin %s failed!!!" %
_strPluginName)
else:
self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def __init__(self, _strPluginName):
"""
Constructor of the class
@param strPluginName: name of the plugin
@type strPluginName: string
"""
EDLogging.__init__(self)
self.__strPluginName = _strPluginName
self.__edPlugin = None
self.__edSlotCallBack = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__pathXSDInput = None
self.__pathXSDOutput = None
self.__bXmlInputSet = False
self.__status = None
self.__name = None
self.__runtime = None
self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(
self.__strPluginName)
if self.__edPlugin is None:
raise RuntimeError("Unable to create plugin %s" %
self.__strPluginName)
self.__jobId = "%s-%08i" % (self.__strPluginName,
self.__edPlugin.getId())
with self.__class__.__semaphore:
self.__class__.__dictJobs[self.__jobId] = self
if (self.__edPlugin is None):
self.WARNING("Instantiation of plugin %s failed!!!" %
_strPluginName)
else:
self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def executePluginSynchronous(self, _edPlugin):
"""
This method is used to start executable plugins in pipeline synchronously.
"""
_edControlSlotSUCCESS = EDSlot()
_edControlSlotFAILURE = EDSlot()
map(_edControlSlotSUCCESS.connect, _edPlugin.getSlotSUCCESS().getListMethod())
map(_edControlSlotFAILURE.connect, _edPlugin.getSlotFAILURE().getListMethod())
_edPlugin.getSlotSUCCESS().emptyListMethod()
_edPlugin.getSlotFAILURE().emptyListMethod()
_edPlugin.executeSynchronous()
if (not _edPlugin.isFailure()):
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotSUCCESS")
# Check that something doesn't go wrong in the success method!
try:
_edControlSlotSUCCESS.call(_edPlugin)
except Exception:
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous: ERROR in slotSUCCESS!")
EDVerbose.writeErrorTrace()
_edPlugin.setFailure()
if (_edPlugin.isFailure()):
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotFAILURE")
# Check that something doesn't go wrong in the success method!
try:
_edControlSlotFAILURE.call(_edPlugin)
except Exception:
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous: ERROR in slotFAILURE!")
EDVerbose.writeErrorTrace()
def executePluginSynchronous(self, _edPlugin):
"""
This method is used to start executable plugins in pipeline synchronously.
"""
_edControlSlotSUCCESS = EDSlot()
_edControlSlotFAILURE = EDSlot()
map(_edControlSlotSUCCESS.connect,
_edPlugin.getSlotSUCCESS().getListMethod())
map(_edControlSlotFAILURE.connect,
_edPlugin.getSlotFAILURE().getListMethod())
_edPlugin.getSlotSUCCESS().emptyListMethod()
_edPlugin.getSlotFAILURE().emptyListMethod()
_edPlugin.executeSynchronous()
if (not _edPlugin.isFailure()):
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotSUCCESS")
# Check that something doesn't go wrong in the success method!
try:
_edControlSlotSUCCESS.call(_edPlugin)
except Exception:
EDVerbose.DEBUG(
"EDControlPlugin.executeSynchronous: ERROR in slotSUCCESS!"
)
EDVerbose.writeErrorTrace()
_edPlugin.setFailure()
if (_edPlugin.isFailure()):
EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotFAILURE")
# Check that something doesn't go wrong in the success method!
try:
_edControlSlotFAILURE.call(_edPlugin)
except Exception:
EDVerbose.DEBUG(
"EDControlPlugin.executeSynchronous: ERROR in slotFAILURE!"
)
EDVerbose.writeErrorTrace()
def __init__(self):
EDLogging.__init__(self)
Thread.__init__(self)
self.__edSlotPreProcess = EDSlot()
self.__edSlotProcess = EDSlot()
self.__edSlotPostProcess = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__edSlotFinallyProcess = EDSlot()
self.__bIsFailure = False
self.__bIsTimeOut = False
self.__fTimeOutInSeconds = None
self.__fDefaultTimeOutInSeconds = 600.0
self.__bIsAbort = False
# Reference to the object which calls execute or executeSynchronous
self.__edObject = None
self.__lExtraTime = [] # list of extra allowed time for execution (in second)
self.__bLogTiming = False
class EDPlugin(EDAction):
"""
This is the EDNA plugin main class
An EDNA plugin class:
- is a configurable entity
- has a base name (<date>-<random number>-<base name>)
- handles input/output data (setter, getter, checker)
- has warning and error messages
- has a base and a working directory (both are configurable)
The working directory is the folder from which the plugin is launched
and should contain all associated files with the plugin execution (edna xml input/output, 3rd party output files)
The base directory is the parent directory of the working directory.
Example: the working directory of a control plugin is the base directory of the plugins that it invokes,
i.e. the plugins working directories has the control plugin working directory as parent.
- defines the method that generates an executive summary (user-related output summary) that sub-classes should implement
"""
CONF_BASE_DIR_LABEL = "baseDirectory"
CONF_WORKING_DIR_LABEL = "workingDirectory"
CONF_TIME_OUT = "timeOut"
CONF_WRITE_XML_INPUT_OUTPUT = "writeXMLInputOutput"
CONF_WRITE_XML_OUTPUT = "writeXMLOutput"
CONF_WRITE_XML_INPUT = "writeXMLInput"
def __init__ (self):
"""
Initializes plugin related attributes described above
"""
EDAction.__init__(self)
self.__xsPluginItem = None
self.__dictXSDataInputClass = {}
self.__dictXSDataInput = {}
self.__dictXSDataOutput = {}
self.__strDefaultInputDataKey = "defaultInputData"
self.__strDefaultOutputDataKey = "defaultOutputData"
self.__edSlotExportDataOutput = EDSlot()
self.__strBaseDirectory = None
self.__strWorkingDirectory = None
self.__strBaseName = None
self.__listExecutiveSummaryLines = []
self.__strExecutiveSummarySeparator = "-" * 80
self.__listErrorMessages = []
self.__listWarningMessages = []
self.__isRequiredToHaveConfiguration = False
self.__bWriteDataXMLInputOutput = True
self.__bWriteDataXMLOutput = True
self.__bWriteDataXMLInput = True
self.__strPluginId = "%s-%08i" % (self.getClassName(), self.getId())
self.strPathDataInput = None
self.strPathDataOutput = None
def preProcess(self, _edObject=None):
"""
Writes xml data input in the working dir (if required)
Connects a slot for generating the executive summary after the plugin execution
Connects a slot for checking output data to the finally process
Initialize the base directory
Configures the plugin
Checks the input data
"""
EDAction.preProcess(self, _edObject)
self.DEBUG("EDPlugin.preProcess")
self.connectPostProcess(self.exportDataOutput)
if self.__bWriteDataXMLInputOutput:
if self.__bWriteDataXMLInput:
self.connectPreProcess(self.writeDataInput)
self.connectPostProcess(self.generateExecutiveSummary)
self.connectFinallyProcess(self.checkDataOutput)
if (self.__strBaseName is None):
self.setBaseName(self.createBaseName())
EDStatus.tellRunning(self.__strPluginId)
self.connectFinallyProcess(self.tellFinished)
self.checkParameters()
def tellFinished(self, _edObject=None):
"""
Tell EDStatus that the plugin has finished, either in success either in error
"""
if self.isFailure():
EDStatus.tellFailure(self.__strPluginId)
else:
EDStatus.tellSuccess(self.__strPluginId)
def checkDataOutput(self, _edObject=None):
"""
Checks if output data is available, if not issues a warning and sets an empty XSDataResult as output data
Writes xml data output in the working dir (if required)
"""
EDAction.finallyProcess(self, _edObject)
if self.__dictXSDataOutput == {}:
strWarningMessage = "Output data for plugin %s not set, using XSDataResult as output" % self.getPluginName()
self.WARNING(strWarningMessage)
self.addWarningMessage(strWarningMessage)
self.setDataOutput(XSDataResult())
if self.__bWriteDataXMLInputOutput:
if self.__bWriteDataXMLOutput:
self.writeDataOutput()
def synchronize(self):
"""
This method calls EDAction.synchronize and if a time-out occurs an error message
is added to the list of error messages.
"""
EDAction.synchronize(self)
if self.isTimeOut():
strErrorMessage = "Timeout when waiting for %s to terminate." % self.getClassName()
self.addErrorMessage(strErrorMessage)
def setConfiguration(self, _xsPluginItem):
"""
Receives an XSPluginItem (Plugin Configuration) from the application
"""
self.DEBUG("EDPlugin.setConfiguration")
self.__xsPluginItem = _xsPluginItem
def getConfiguration(self):
"""
Gets the Plugin Configuration as an XSPluginItem
"""
self.DEBUG("EDPlugin.getConfiguration")
return self.__xsPluginItem
configuration = property(getConfiguration, setConfiguration)
def getStringConfigurationParameterValue(self, _strConfigurationParameterName):
"""
This method returns a configuration parameter value if a corresponding configuration
parameter name can be found in the configuration file.
If an application wide configuration file is provided via EDApplication it will
override the product configuration file.
The configuration parameter is then searched in the configration file in following order:
- If a plugin configuration item exists and the configration parameter name is present it will be used.
- Otherwise if a product-wide (e.g. "mxPluginExec") configuration value exists it will be used.
"""
self.DEBUG("EDPlugin.getConfigurationParameterValue")
strParameteValue = None
xsPluginItem = self.getConfiguration()
bFoundConfigurationParameter = False
if xsPluginItem:
strParameteValue = EDConfiguration.getParamItem(xsPluginItem, _strConfigurationParameterName).getValue()
if strParameteValue:
bFoundConfigurationParameter = True
if not bFoundConfigurationParameter:
xsPluginItem = EDApplication.getApplicationPluginConfiguration(self.getPluginName())
if xsPluginItem:
strParameteValue = EDConfiguration.getParamItem(xsPluginItem, _strConfigurationParameterName).getValue()
if strParameteValue:
bFoundConfigurationParameter = True
if not bFoundConfigurationParameter:
xsPluginItem = EDApplication.getProjectPluginConfiguration(self.getPluginName())
if xsPluginItem:
strParameteValue = EDConfiguration.getParamItem(xsPluginItem, _strConfigurationParameterName).getValue()
if strParameteValue:
bFoundConfigurationParameter = True
return strParameteValue
def getDoubleConfigurationParameterValue(self, _strConfigurationParameterName):
fParameterValue = None
strParameterValue = self.getStringConfigurationParameterValue(_strConfigurationParameterName)
if strParameterValue:
fParameterValue = float(strParameterValue)
return fParameterValue
def getIntegerConfigurationParameterValue(self, _strConfigurationParameterName):
iParameterValue = None
strParameterValue = self.getStringConfigurationParameterValue(_strConfigurationParameterName)
if strParameterValue:
iParameterValue = int(strParameterValue)
return iParameterValue
def configure(self):
"""
Should be overridden by the Final Plugin If needed
This method should set its proper members attributes from a Plugin configuration Object
"""
self.DEBUG("EDPlugin.configure : %s" % self.getClassName())
xsPluginItem = self.getConfiguration()
if xsPluginItem is None:
xsPluginItem = EDApplication.getApplicationPluginConfiguration(self.getPluginName())
if (xsPluginItem is None):
# No application wide configuration file found! Try to find a project specific config file:
xsPluginItem = EDApplication.getProjectPluginConfiguration(self.getPluginName())
if (xsPluginItem is None):
self.DEBUG("EDPlugin.configure: No plugin configuration found for " + self.getPluginName())
xsPluginItem = XSPluginItem()
else:
self.setConfiguration(xsPluginItem)
# set Timeout if different from default one
if self.getTimeOut() == self.getDefaultTimeOut():
# Try to get time out from plugin configuration
iTimeOut = EDConfiguration.getIntegerParamValue(xsPluginItem, EDPlugin.CONF_TIME_OUT)
if iTimeOut is not None:
self.DEBUG("EDPlugin.configure: Setting time out to %d s from plugin configuration." % iTimeOut)
self.setTimeOut(iTimeOut)
else:
self.DEBUG("EDPlugin.configure: Working directory already set before plugin is configured.")
# Base directory
strBaseDirectory = self.getBaseDirectory()
if (strBaseDirectory is None):
# Try to get base directory from plugin configuration
strBaseDirectory = EDConfiguration.getStringParamValue(xsPluginItem, EDPlugin.CONF_BASE_DIR_LABEL)
if(strBaseDirectory is None):
# Try to get working directory from environment variable
strBaseDirectory = os.environ.get("EDNA_BASE_DIRECTORY")
if (strBaseDirectory is None):
self.DEBUG("EDPlugin.configure: Using current base directory as working directory.")
strBaseDirectory = os.getcwd()
else:
self.DEBUG("EDPlugin.configure: Setting base directory from $EDNA_WORKING_DIRECTORY.")
else:
if (strBaseDirectory == "."):
self.DEBUG("EDPlugin.configure: Using current base directory as working directory.")
strBaseDirectory = os.getcwd()
else:
strBaseDirectory = os.path.abspath(strBaseDirectory)
self.DEBUG("EDPlugin.configure: Setting base directory from plugin configuration.")
self.setBaseDirectory(strBaseDirectory)
else:
self.DEBUG("EDPlugin.configure: Base directory already set before plugin is configured.")
# Working directory
strWorkingDirectory = self.getWorkingDirectory()
if (strWorkingDirectory is None):
# Try to get working directory from plugin configuration
strWorkingDirectory = EDConfiguration.getStringParamValue(xsPluginItem, EDPlugin.CONF_WORKING_DIR_LABEL)
if(strWorkingDirectory is not None):
self.DEBUG("EDPlugin.configure: Setting working directory from plugin configuration.")
else:
self.DEBUG("EDPlugin.configure: Setting working directory as base directory + base name.")
strWorkingDirectory = os.path.join(self.getBaseDirectory(), self.getBaseName())
self.setWorkingDirectory(strWorkingDirectory)
else:
self.DEBUG("EDPlugin.configure: Working directory already set before plugin is configured.")
#
strWriteXMLInputOutput = EDConfiguration.getStringParamValue(xsPluginItem, EDPlugin.CONF_WRITE_XML_INPUT_OUTPUT)
if strWriteXMLInputOutput != None:
if strWriteXMLInputOutput.lower().strip() in ["false", "0"]:
self.__bWriteDataXMLInputOutput = False
else:
self.__bWriteDataXMLInputOutput = True
strWriteXMLOutput = EDConfiguration.getStringParamValue(xsPluginItem, EDPlugin.CONF_WRITE_XML_OUTPUT)
if strWriteXMLOutput != None:
if strWriteXMLOutput.lower().strip() in ["false", "0"]:
self.__bWriteDataXMLOutput = False
else:
self.__bWriteDataXMLOutput = True
strWriteXMLInput = EDConfiguration.getStringParamValue(xsPluginItem, EDPlugin.CONF_WRITE_XML_INPUT)
if strWriteXMLInput != None:
if strWriteXMLInput.lower().strip() in ["false", "0"]:
self.__bWriteDataXMLInput = False
else:
self.__bWriteDataXMLInput = True
def execute(self, _edObject=None):
# Configure the plugin before starting it's thread
self.configure()
EDAction.execute(self, _edObject)
def checkParameters(self):
"""
Should be overridden by the Final Plugin If needed
This method should check that the data input are consistent
"""
self.DEBUG("EDPlugin.checkParameters")
def setXSDataInputClass(self, _xsDataInputClass, _strDataInputKey=None):
"""
This method should be called in the constructor of the derived plugins
in order to set the XSData type of the input data, e.g. XSDataInputXXX
"""
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInputClass.keys()):
strErrorMessage = "ERROR: " + self.getPluginName() + ".setXSDataInputClass, Data Input Class already defined for key: " + strDataInputKey
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError, strErrorMessage
self.__dictXSDataInputClass[ strDataInputKey ] = _xsDataInputClass
def getXSDataInputClass(self, _strDataInputKey=None):
"""
Returns the XSData type of the input data.
"""
pyXSDataInputClass = None
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInputClass.keys()):
pyXSDataInputClass = self.__dictXSDataInputClass[ strDataInputKey ]
return pyXSDataInputClass
def setDataInput(self, _oDataInput, _strDataInputKey=None):
"""
Sets the plugin input data.
_oDataInput could be either an String XML or an XSData object.
The input data is stored in a dictionary with the key _strDataInputKey.
If the key is not provided a default key is used.
If not data input class is defined for the key an exception is raised.
If the key is not the default key, the data object is added to a list which
might contain already stored object(s).
If _oDataInput is None the list corresponding to a keyword is deleted.
"""
self.DEBUG("EDPlugin.setDataInput")
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
# Allow for None input
if (_oDataInput is None):
self.DEBUG("EDPlugin.setDataInput: Data input is None")
self.__dictXSDataInput[ strDataInputKey ] = []
elif (self.getXSDataInputClass(strDataInputKey) is None):
strErrorMessage = "ERROR: " + self.getPluginName() + ".setDataInput, Data Input Class not defined for key: " + strDataInputKey
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError, strErrorMessage
else:
# Check the type
xsDataInput = None
if isinstance(_oDataInput, (str, unicode)):
self.DEBUG("EDPlugin.setDataInput: Input Data is string ")
xsDataInput = self.getXSDataInputClass(strDataInputKey).parseString(_oDataInput)
elif (isinstance(_oDataInput, self.getXSDataInputClass(strDataInputKey))):
self.DEBUG("EDPlugin.setDataInput: Input Data is of type " + str(_oDataInput.__class__))
xsDataInput = _oDataInput
else:
strErrorMessage = "ERROR: %s.setDataInput, wrong data type %r for data input key: %s, expected XML string or %r" % \
(self.getPluginName(), _oDataInput.__class__, strDataInputKey, self.getXSDataInputClass(strDataInputKey))
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError, strErrorMessage
# Add the object to a list if its key is not the default key
if (strDataInputKey != self.__strDefaultInputDataKey) :
# Check if there's already a list stored
if (not strDataInputKey in self.__dictXSDataInput.keys()):
self.__dictXSDataInput[ strDataInputKey ] = []
self.__dictXSDataInput[ strDataInputKey ].append(xsDataInput)
else:
self.__dictXSDataInput[ strDataInputKey ] = xsDataInput
def hasDataInput(self, _strDataInputKey=None):
"""
Returns True if the plugin has Input Data for a particular key.
If the key is not provided a default key is used.
"""
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInput.keys()):
return True
else:
return False
def getDataInput(self, _strDataInputKey=None):
"""
Returns the Plugin Input Data for a particular key.
If the key is not provided a default key is used.
"""
oValue = None
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInput.keys()):
oValue = self.__dictXSDataInput[ strDataInputKey ]
else:
strErrorMessage = self.getPluginName() + ".getDataInput, no input data defined for key: " + strDataInputKey
self.warning(strErrorMessage)
self.addWarningMessage(strErrorMessage)
return oValue
def delDataInput(self, _strDataInputKey=None):
"""
Deletes the data input for a particular key.
If the key is not provided a default key is used.
"""
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInput.keys()):
self.__dictXSDataInput[ strDataInputKey ] = None
else:
strErrorMessage = self.getPluginName() + ".delDataInput, no input data defined for key: " + strDataInputKey
self.warning(strErrorMessage)
self.addWarningMessage(strErrorMessage)
# Property for dataInput
dataInput = property(getDataInput, setDataInput, delDataInput, "Property for dataInput")
def setDataOutput(self, _xsDataOutput, _strDataOutputKey=None):
"""
Sets the plugin output data for a particular key.
If the key is not provided a default key is used.
If the key is already defined in the dictionary, the corresponding
data object is added to a list which contains the already stored object(s).
"""
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
# Add the object to a list if its key not the default key
if (strDataOutputKey == self.__strDefaultOutputDataKey):
self.__dictXSDataOutput[ strDataOutputKey ] = _xsDataOutput
else:
# Check if the _xsDataoutput object is already a list
if (type(_xsDataOutput) == type ([])):
self.__dictXSDataOutput[ strDataOutputKey ] = _xsDataOutput
else:
# Check if the stored object contains already a list
if (not strDataOutputKey in self.__dictXSDataOutput.keys()):
self.__dictXSDataOutput[ strDataOutputKey ] = []
self.__dictXSDataOutput[ strDataOutputKey ].append(_xsDataOutput)
def getDataOutput(self, _strDataOutputKey=None):
"""
Returns the Plugin Output Data
"""
oValue = None
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
if (strDataOutputKey in self.__dictXSDataOutput.keys()):
oValue = self.__dictXSDataOutput[ strDataOutputKey ]
return oValue
def hasDataOutput(self, _strDataOutputKey=None):
"""
Returns True if the plugin has the specified Output Data
"""
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
if (strDataOutputKey in self.__dictXSDataOutput.keys()):
return True
else:
return False
def delDataOutput(self, _strDataOutputKey=None):
"""
Deletes the data output for a particular key.
If the key is not provided a default key is used.
"""
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
if (strDataOutputKey in self.__dictXSDataOutput.keys()):
self.__dictXSDataOutput[ strDataOutputKey ] = None
else:
strErrorMessage = self.getPluginName() + ".delDataOutput, no output data defined for key: " + strDataInputKey
self.warning(strErrorMessage)
self.addWarningMessage(strErrorMessage)
# Property for dataOutput
dataOutput = property(getDataOutput, setDataOutput, delDataOutput, "Property for dataOutput")
def exportDataOutput(self, _edPlugin=None):
"""
Deprecated
Exports the Plugin Output Data to slot
"""
self.DEBUG("EDPlugin.exportDataOutput")
self.__edSlotExportDataOutput.call(self.__dictXSDataOutput)
def connectExportDataOutput(self, _oMethod):
"""
Deprecated
"""
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotExportDataOutput.connect(_oMethod)
self.synchronizeOff()
def generateExecutiveSummary(self, _edPlugin):
"""
This method, which should be implemented by sub-classes, generates an executive summary (user-related output summary).
"""
self.DEBUG("EDPlugin.generateExecutiveSummary")
def addErrorMessage(self, _strErrorMessage):
"""
Adds an error message to the error messages list
"""
self.__listErrorMessages.append(_strErrorMessage)
def getErrorMessages(self):
"""
Returns the error messages list
OBS! This method is deprecated, please use getListOfErrorMessages instead.
"""
self.warning("Deprecation by Monday 7th June 2010 of EDPlugin, called getErrorMessages")
from EDImportLib import EDList
return EDList(self.__listErrorMessages)
def getListOfErrorMessages(self):
"""
Returns the error messages list
"""
return self.__listErrorMessages
def addWarningMessage(self, _strWarningMessage):
"""
Adds a warning message to the warning messages list
"""
self.DEBUG("EDPlugin.addWarningMessage : " + _strWarningMessage)
self.__listWarningMessages.append(_strWarningMessage)
def getWarningMessages(self):
"""
Returns the warning messages list
OBS! This method is deprecated, please use getListOfWarningMessages instead.
"""
self.warning("Deprecation by Monday 7th June 2010 of EDPlugin, called getWarningMessages")
from EDImportLib import EDList
return EDList(self.__listWarningMessages)
def getListOfWarningMessages(self):
"""
Returns the warning messages list
"""
return self.__listWarningMessages
def writeDataInput(self, _edObject=None):
"""
Writes the input data object(s) into a working dir xml file
"""
self.DEBUG("EDPlugin.writeDataInput")
strBasename = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()))
for strKey in self.__dictXSDataInput.keys():
if (strKey == self.__strDefaultInputDataKey): # "Old" style
xsDataInput = self.__dictXSDataInput[ self.__strDefaultInputDataKey ]
self.strPathDataInput = strBasename + "_dataInput.xml"
EDUtilsFile.writeFile(self.strPathDataInput, xsDataInput.marshal())
else: # We have a list of objects
listXSDataInput = self.__dictXSDataInput[ strKey ]
for iIndex, xsDataInput in enumerate(listXSDataInput):
strPathDataInput = "%s_%s_%d_dataInput.xml" % (strBasename, strKey, iIndex)
EDUtilsFile.writeFile(strPathDataInput, xsDataInput.marshal())
def writeDataOutput(self, _edObject=None):
"""
Writes the output data object(s) into a working dir xml file
"""
self.DEBUG("EDPlugin.writeDataOutput")
for strKey in self.__dictXSDataOutput.keys():
if (strKey == self.__strDefaultOutputDataKey): # "Old" style
xsDataOutput = self.__dictXSDataOutput[ self.__strDefaultOutputDataKey ]
if (xsDataOutput is not None):
self.strPathDataOutput = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()) + "_dataOutput.xml")
EDUtilsFile.writeFile(self.strPathDataOutput, xsDataOutput.marshal())
else:
listXSDataOutput = self.__dictXSDataOutput[ strKey ]
for iIndex, xsDataOutput in enumerate(listXSDataOutput):
if (xsDataOutput is not None):
strPathDataOutput = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()) + "_" + strKey + "_%d_dataOutput.xml" % iIndex)
EDUtilsFile.writeFile(strPathDataOutput, xsDataOutput.marshal())
def getBaseName(self):
"""
Returns the plugin base name
"""
if (self.__strBaseName is None):
self.__strBaseName = self.createBaseName()
return self.__strBaseName
def setBaseName(self, _strBaseName):
"""
Sets the plugin base name
"""
self.__strBaseName = self.compactPluginName(_strBaseName)
self.setName(self.__strBaseName)
# Create the directory baseDirectory/baseName which will be used as working directory
strWorkingDirPath = os.path.join(self.getBaseDirectory(), self.__strBaseName)
if not os.path.isdir(strWorkingDirPath):
os.mkdir(strWorkingDirPath)
self.setWorkingDirectory(strWorkingDirPath)
def createBaseName(self):
"""
Generates the plugin base name: (<prefix>-<object ID>)
"""
# First try to use global instance ID from EDObject
strBaseName = "%s-%08d" % (self.compactPluginName(self.getPluginName()), self.getId())
strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName)
# Try to create the directory...
try:
os.mkdir(strBaseDir)
except BaseException, strErrorDetail:
self.error("EDPlugin.createBaseName: Could not create base directory %s because of %s" % (strBaseDir, strErrorDetail))
self.warning("EDPlugin.createBaseName: Trying to create alternative base directory...")
self.writeErrorTrace()
strTempDir = tempfile.mkdtemp(prefix=strBaseName, dir=self.getBaseDirectory())
os.chmod(strTempDir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
strBaseName = os.path.split(strTempDir)[1]
strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName)
self.warning("EDPlugin.createBaseName: Alternative base directory created: %s" % strBaseDir)
self.DEBUG("EDPlugin.createBaseName : Directory created = " + strBaseDir)
return strBaseName
class EDJob(EDObject):
PLUGIN_STATE_UNITIALIZED = "uninitialized"
PLUGIN_STATE_RUNNING = "running"
PLUGIN_STATE_SUCCESS = "success"
PLUGIN_STATE_FAILURE = "failure"
__edFactoryPlugin = EDFactoryPlugin()
__dictJobs = {}
__semaphore = threading.Semaphore()
__dictPluginLastId = {}
__fStartTime = time.time()
def __init__(self, _strPluginName):
"""
Constructor of the class
@param strPluginName: name of the plugin
@type strPluginName: string
"""
EDObject.__init__(self)
self.__strPluginName = _strPluginName
self.__edPlugin = None
self.__edSlotCallBack = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__bXmlInputSet = False
self.__status = None
# self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName)
self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName)
EDJob.__semaphore.acquire()
# Create the JobID
if not self.__strPluginName in EDJob.__dictPluginLastId:
EDJob.__dictPluginLastId[_strPluginName] = 0
else:
EDJob.__dictPluginLastId[_strPluginName] += 1
self.__jobId = "%s-%i" % (self.__strPluginName, EDJob.__dictPluginLastId[_strPluginName])
EDJob.__dictJobs[self.__jobId] = self
EDJob.__semaphore.release()
if (self.__edPlugin is None):
EDVerbose.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName)
else:
self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def setDataInput(self, _oDataInput, _strDataInputKey=None):
"""
Sets the job (plugin) input data.
@param: _oDataInput: could be either an String XML or an XSData object.
@param _strDataInputKey: the key of an input data dictionnary
The input data is stored in a dictionary with the key _strDataInputKey.
If the key is not provided a default key is used.
If not data input class is defined for the key an exception is raised.
If the key is not the default key, the data object is added to a list which
might contain already stored object(s).
If _oDataInput is None the list corresponding to a keyword is deleted.
"""
if _oDataInput in ["", None]:
self.__bXmlInputSet = False
return
self.synchronizeOn()
if (self.__edPlugin is not None):
self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey)
self.__bXmlInputSet = True
else:
EDVerbose.WARNING("Setting DataInput for uninstanciated plugin %s." % self.__strPluginName)
self.synchronizeOff()
def getDataInput(self, _strDataInputKey=None):
"""
Returns the Plugin Input Data for a particular key.
If the key is not provided a default key is used.
"""
if (self.__edPlugin is not None):
return self.__edPlugin.getDataInput(_strDataInputKey)
else:
EDVerbose.WARNING("Getting DataInput for uninstanciated plugin %s." % self.__strPluginName)
def getDataOutput(self, _strDataOutputKey=None):
"""
Returns the Plugin Output Data
"""
if (self.__edPlugin is not None):
return self.__edPlugin.getDataOutput(_strDataOutputKey)
else:
EDVerbose.WARNING("Getting DataOutput for uninstanciated plugin %s." % self.__strPluginName)
def execute(self):
"""
Launch the EDNA plugin
@return: JobId
@rtype: string
"""
returnId = None
if not self.__bXmlInputSet:
EDVerbose.WARNING("Not executing job %s as input is empty" % self.__jobId)
if (self.__edPlugin is not None):
self.synchronizeOn()
self.__status = EDJob.PLUGIN_STATE_RUNNING
self.__edPlugin.connectSUCCESS(self.successPluginExecution)
self.__edPlugin.connectFAILURE(self.failurePluginExecution)
self.__edPlugin.execute()
returnId = self.__jobId
self.synchronizeOff()
else:
EDVerbose.WARNING("Trying to run a plugin that does not exist: %s " % self.__strPluginName)
return returnId
def synchronize(self):
"""
Synchronize the execution of the job with the calling thread.
"""
self.__edPlugin.synchronize()
def successPluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin succeeds
"""
self.synchronizeOn()
self.__status = EDJob.PLUGIN_STATE_SUCCESS
EDVerbose.screen("Plugin %s execution ended with success" % self.__jobId)
self.synchronizeOff()
try:
self.__edSlotSUCCESS.call(self.__jobId)
except Exception:
EDVerbose.ERROR("Error in execution of Success call-back for %s" % self.__jobId)
EDVerbose.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
EDVerbose.ERROR("Error in execution of Common call-back (after success) for %s" % self.__jobId)
EDVerbose.writeErrorTrace()
def failurePluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin failed
"""
self.synchronizeOn()
self.__status = EDJob.PLUGIN_STATE_FAILURE
EDVerbose.screen("Plugin %s execution ended with failure" % self.__jobId)
self.synchronizeOff()
try:
self.__edSlotFAILURE.call(self.__jobId)
except Exception:
EDVerbose.ERROR("Error in execution of Failure call-back for %s" % self.__jobId)
EDVerbose.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
EDVerbose.ERROR("Error in execution of Common call-back (after failure) for %s" % self.__jobId)
EDVerbose.writeErrorTrace()
def connectSUCCESS(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotSUCCESS.connect(_oMethod)
self.synchronizeOff()
def connectFAILURE(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotFAILURE.connect(_oMethod)
self.synchronizeOff()
def connectCallBack(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotCallBack.connect(_oMethod)
self.synchronizeOff()
def getJobId(self):
"""
@return: JobId i.e. EDPluginName-Number
@rtype: string
"""
return self.__jobId
def getPluginName(self):
"""
@return: Name of the plugin
@rtype: string
"""
return self.__strPluginName
def getPlugin(self):
"""
@return: the plugin (instance)
@rtype: python object
"""
return self.__edPlugin
def getStatus(self):
"""
@return: status of the Job
@rtype: string
"""
return self.__status
def getMemSize(self):
"""
try to guess the size in memory of a job
@return: expected size in memory
"""
if asizeof is not None:
return asizeof.asizeof(self)
@staticmethod
def getStatusFromID(jobId):
"""
Retrieve the job (hence the plugin) status
@param jobId: the Job identification number
@type jobId: string
@return: the EDJob status
@rtype: string
"""
if jobId in EDJob.__dictJobs:
return EDJob.__dictJobs[jobId].getStatus()
else:
EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId)
@staticmethod
def getJobFromID(jobId):
"""
Retrieve the job (hence the plugin)
@param jobId: the Job identification number
@type jobId: string
@return: the "EDJob instance", which contains the plugin (__edPlugin) and the status
@rtype: a Python object.
"""
if jobId in EDJob.__dictJobs:
return EDJob.__dictJobs[jobId]
else:
EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId)
@staticmethod
def getMemoryFootprint():
if asizeof is not None:
return asizeof.asizesof(EDJob.__dictJobs)
@staticmethod
def stats():
"""
retrieve some statistics
"""
output = []
fExecTime = time.time() - EDJob.__fStartTime
keys = EDJob.__dictJobs.keys()
for key in keys :
num = int(key.split("-", 1)[1])
job = EDJob.__dictJobs[key]
output.append([num, job.getPluginName(), job.getStatus(), job.getPlugin().getRunTime(), job.getMemSize()])
output.sort()
iNbJob = max(1, len(output))
EDVerbose.screen("%s\t|\t%s\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("id", "EDPluginName", "status", "runtime", "memory"))
fWall = 0.0
fSumProd = 0.0
fSumX = 0.0
fSumX2 = 0.0
for oneJob in output:
fWall += oneJob[3]
fSumX += oneJob[0]
fSumX2 += oneJob[0] * oneJob[0]
fSumProd += oneJob[0] * oneJob[3]
EDVerbose.screen("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob))
EDVerbose.screen("_" * 90)
EDVerbose.screen("Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime))
EDVerbose.screen("Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob))
fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX)
fOrd = (fWall - fSlope * fSumX) / iNbJob
EDVerbose.screen("Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope))
class EDJob(EDObject):
PLUGIN_STATE_UNITIALIZED = "uninitialized"
PLUGIN_STATE_RUNNING = "running"
PLUGIN_STATE_SUCCESS = "success"
PLUGIN_STATE_FAILURE = "failure"
__edFactoryPlugin = EDFactoryPlugin()
__dictJobs = {}
__semaphore = threading.Semaphore()
__dictPluginLastId = {}
__fStartTime = time.time()
def __init__(self, _strPluginName):
"""
Constructor of the class
@param strPluginName: name of the plugin
@type strPluginName: string
"""
EDObject.__init__(self)
self.__strPluginName = _strPluginName
self.__edPlugin = None
self.__edSlotCallBack = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__bXmlInputSet = False
self.__status = None
# self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName)
self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName)
EDJob.__semaphore.acquire()
# Create the JobID
if not self.__strPluginName in EDJob.__dictPluginLastId:
EDJob.__dictPluginLastId[_strPluginName] = 0
else:
EDJob.__dictPluginLastId[_strPluginName] += 1
self.__jobId = "%s-%i" % (self.__strPluginName,
EDJob.__dictPluginLastId[_strPluginName])
EDJob.__dictJobs[self.__jobId] = self
EDJob.__semaphore.release()
if (self.__edPlugin is None):
EDVerbose.WARNING("Instantiation of plugin %s failed!!!" %
_strPluginName)
else:
self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def setDataInput(self, _oDataInput, _strDataInputKey=None):
"""
Sets the job (plugin) input data.
@param: _oDataInput: could be either an String XML or an XSData object.
@param _strDataInputKey: the key of an input data dictionnary
The input data is stored in a dictionary with the key _strDataInputKey.
If the key is not provided a default key is used.
If not data input class is defined for the key an exception is raised.
If the key is not the default key, the data object is added to a list which
might contain already stored object(s).
If _oDataInput is None the list corresponding to a keyword is deleted.
"""
if _oDataInput in ["", None]:
self.__bXmlInputSet = False
return
self.synchronizeOn()
if (self.__edPlugin is not None):
self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey)
self.__bXmlInputSet = True
else:
EDVerbose.WARNING(
"Setting DataInput for uninstanciated plugin %s." %
self.__strPluginName)
self.synchronizeOff()
def getDataInput(self, _strDataInputKey=None):
"""
Returns the Plugin Input Data for a particular key.
If the key is not provided a default key is used.
"""
if (self.__edPlugin is not None):
return self.__edPlugin.getDataInput(_strDataInputKey)
else:
EDVerbose.WARNING(
"Getting DataInput for uninstanciated plugin %s." %
self.__strPluginName)
def getDataOutput(self, _strDataOutputKey=None):
"""
Returns the Plugin Output Data
"""
if (self.__edPlugin is not None):
return self.__edPlugin.getDataOutput(_strDataOutputKey)
else:
EDVerbose.WARNING(
"Getting DataOutput for uninstanciated plugin %s." %
self.__strPluginName)
def execute(self):
"""
Launch the EDNA plugin
@return: JobId
@rtype: string
"""
returnId = None
if not self.__bXmlInputSet:
EDVerbose.WARNING("Not executing job %s as input is empty" %
self.__jobId)
if (self.__edPlugin is not None):
self.synchronizeOn()
self.__status = EDJob.PLUGIN_STATE_RUNNING
self.__edPlugin.connectSUCCESS(self.successPluginExecution)
self.__edPlugin.connectFAILURE(self.failurePluginExecution)
self.__edPlugin.execute()
returnId = self.__jobId
self.synchronizeOff()
else:
EDVerbose.WARNING(
"Trying to run a plugin that does not exist: %s " %
self.__strPluginName)
return returnId
def synchronize(self):
"""
Synchronize the execution of the job with the calling thread.
"""
self.__edPlugin.synchronize()
def successPluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin succeeds
"""
self.synchronizeOn()
self.__status = EDJob.PLUGIN_STATE_SUCCESS
EDVerbose.screen("Plugin %s execution ended with success" %
self.__jobId)
self.synchronizeOff()
try:
self.__edSlotSUCCESS.call(self.__jobId)
except Exception:
EDVerbose.ERROR("Error in execution of Success call-back for %s" %
self.__jobId)
EDVerbose.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
EDVerbose.ERROR(
"Error in execution of Common call-back (after success) for %s"
% self.__jobId)
EDVerbose.writeErrorTrace()
def failurePluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin failed
"""
self.synchronizeOn()
self.__status = EDJob.PLUGIN_STATE_FAILURE
EDVerbose.screen("Plugin %s execution ended with failure" %
self.__jobId)
self.synchronizeOff()
try:
self.__edSlotFAILURE.call(self.__jobId)
except Exception:
EDVerbose.ERROR("Error in execution of Failure call-back for %s" %
self.__jobId)
EDVerbose.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
EDVerbose.ERROR(
"Error in execution of Common call-back (after failure) for %s"
% self.__jobId)
EDVerbose.writeErrorTrace()
def connectSUCCESS(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotSUCCESS.connect(_oMethod)
self.synchronizeOff()
def connectFAILURE(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotFAILURE.connect(_oMethod)
self.synchronizeOff()
def connectCallBack(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotCallBack.connect(_oMethod)
self.synchronizeOff()
def getJobId(self):
"""
@return: JobId i.e. EDPluginName-Number
@rtype: string
"""
return self.__jobId
def getPluginName(self):
"""
@return: Name of the plugin
@rtype: string
"""
return self.__strPluginName
def getPlugin(self):
"""
@return: the plugin (instance)
@rtype: python object
"""
return self.__edPlugin
def getStatus(self):
"""
@return: status of the Job
@rtype: string
"""
return self.__status
def getMemSize(self):
"""
try to guess the size in memory of a job
@return: expected size in memory
"""
if asizeof is not None:
return asizeof.asizeof(self)
@staticmethod
def getStatusFromID(jobId):
"""
Retrieve the job (hence the plugin) status
@param jobId: the Job identification number
@type jobId: string
@return: the EDJob status
@rtype: string
"""
if jobId in EDJob.__dictJobs:
return EDJob.__dictJobs[jobId].getStatus()
else:
EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId)
@staticmethod
def getJobFromID(jobId):
"""
Retrieve the job (hence the plugin)
@param jobId: the Job identification number
@type jobId: string
@return: the "EDJob instance", which contains the plugin (__edPlugin) and the status
@rtype: a Python object.
"""
if jobId in EDJob.__dictJobs:
return EDJob.__dictJobs[jobId]
else:
EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId)
@staticmethod
def getMemoryFootprint():
if asizeof is not None:
return asizeof.asizesof(EDJob.__dictJobs)
@staticmethod
def stats():
"""
retrieve some statistics
"""
output = []
fExecTime = time.time() - EDJob.__fStartTime
keys = EDJob.__dictJobs.keys()
for key in keys:
num = int(key.split("-", 1)[1])
job = EDJob.__dictJobs[key]
output.append([
num,
job.getPluginName(),
job.getStatus(),
job.getPlugin().getRunTime(),
job.getMemSize()
])
output.sort()
iNbJob = max(1, len(output))
EDVerbose.screen("%s\t|\t%s\t\t\t|\t%s\t|\t%s\t\t|\t%s" %
("id", "EDPluginName", "status", "runtime", "memory"))
fWall = 0.0
fSumProd = 0.0
fSumX = 0.0
fSumX2 = 0.0
for oneJob in output:
fWall += oneJob[3]
fSumX += oneJob[0]
fSumX2 += oneJob[0] * oneJob[0]
fSumProd += oneJob[0] * oneJob[3]
EDVerbose.screen("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" %
tuple(oneJob))
EDVerbose.screen("_" * 90)
EDVerbose.screen(
"Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f"
% (fWall, fExecTime, fWall / fExecTime))
EDVerbose.screen(
"Average execution time (Wall/N): %.3fs, Average throughput: %.3fs"
% (fWall / iNbJob, fExecTime / iNbJob))
fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 -
fSumX * fSumX)
fOrd = (fWall - fSlope * fSumX) / iNbJob
EDVerbose.screen(
"Regression of execution time: ExecTime = %.3f + %f * NbJob" %
(fOrd, fSlope))
class EDPlugin(EDAction):
"""
This is the EDNA plugin main class
An EDNA plugin class:
- is a configurable entity
- has a base name (<date>-<random number>-<base name>)
- handles input/output data (setter, getter, checker)
- has warning and error messages
- has a base and a working directory (both are configurable)
The working directory is the folder from which the plugin is launched
and should contain all associated files with the plugin execution (edna xml input/output, 3rd party output files)
The base directory is the parent directory of the working directory.
Example: the working directory of a control plugin is the base directory of the plugins that it invokes,
i.e. the plugins working directories has the control plugin working directory as parent.
- defines the method that generates an executive summary (user-related output summary) that sub-classes should implement
"""
CONF_BASE_DIR_LABEL = "baseDirectory"
CONF_WORKING_DIR_LABEL = "workingDirectory"
CONF_TIME_OUT = "timeOut"
CONF_WRITE_XML_INPUT_OUTPUT = "writeXMLInputOutput"
CONF_WRITE_XML_OUTPUT = "writeXMLOutput"
CONF_WRITE_XML_INPUT = "writeXMLInput"
def __init__ (self):
"""
Initializes plugin related attributes described above
"""
EDAction.__init__(self)
self.__xsPluginItem = None
self.__dictXSDataInputClass = {}
self.__dictXSDataInput = {}
self.__dictXSDataOutput = {}
self.__strDefaultInputDataKey = "defaultInputData"
self.__strDefaultOutputDataKey = "defaultOutputData"
self.__edSlotExportDataOutput = EDSlot()
self.__strBaseDirectory = None
self.__strWorkingDirectory = None
self.__strBaseName = None
self.__listExecutiveSummaryLines = []
self.__strExecutiveSummarySeparator = "-" * 80
self.__listErrorMessages = []
self.__listWarningMessages = []
self.__isRequiredToHaveConfiguration = False
self.__bWriteDataXMLInputOutput = True
self.__bWriteDataXMLOutput = True
self.__bWriteDataXMLInput = True
self.__strPluginId = "%s-%08i" % (self.getClassName(), self.getId())
self.strPathDataInput = None
self.strPathDataOutput = None
self.__bUseWarningInsteadOfError = False
self.__edConfiguration = EDConfigurationStatic()
def preProcess(self, _edObject=None):
"""
Writes xml data input in the working dir (if required)
Connects a slot for generating the executive summary after the plugin execution
Connects a slot for checking output data to the finally process
Initialize the base directory
Configures the plugin
Checks the input data
"""
EDAction.preProcess(self, _edObject)
self.DEBUG("EDPlugin.preProcess")
self.connectPostProcess(self.exportDataOutput)
if self.__bWriteDataXMLInputOutput:
if self.__bWriteDataXMLInput:
self.connectPreProcess(self.writeDataInput)
self.connectPostProcess(self.generateExecutiveSummary)
self.connectFinallyProcess(self.checkDataOutput)
if (self.__strBaseName is None):
self.setBaseName(self.createBaseName())
EDStatus.tellRunning(self.__strPluginId)
self.connectFinallyProcess(self.tellFinished)
self.checkParameters()
def tellFinished(self, _edObject=None):
"""
Tell EDStatus that the plugin has finished, either in success either in error
"""
if self.isFailure():
EDStatus.tellFailure(self.__strPluginId)
else:
EDStatus.tellSuccess(self.__strPluginId)
def checkDataOutput(self, _edObject=None):
"""
Checks if output data is available, if not issues a warning and sets an empty XSDataResult as output data
Writes xml data output in the working dir (if required)
"""
EDAction.finallyProcess(self, _edObject)
if self.__dictXSDataOutput == {}:
strWarningMessage = "Output data for plugin %s not set, using XSDataResult as output" % self.getPluginName()
self.WARNING(strWarningMessage)
self.addWarningMessage(strWarningMessage)
self.setDataOutput(XSDataResult())
if self.__bWriteDataXMLInputOutput:
if self.__bWriteDataXMLOutput:
self.writeDataOutput()
def synchronize(self):
"""
This method calls EDAction.synchronize and if a time-out occurs an error message
is added to the list of error messages.
"""
EDAction.synchronize(self)
if self.isTimeOut():
strErrorMessage = "Timeout when waiting for %s to terminate." % self.getClassName()
self.addErrorMessage(strErrorMessage)
def getConfig(self):
"""
Gets the Plugin Configuration as a dictionary
"""
# self.DEBUG("EDPlugin.getConfig")
return self.__edConfiguration.get(self.getPluginName(), {})
def setConfig(self, _dict, _bLocal=False):
"""
Receives a dictionary (Plugin Configuration) from the application.
"""
self.DEBUG("EDPlugin.setConfiguration")
if _bLocal:
self.__edConfiguration = EDConfiguration()
if _dict is not None:
self.__edConfiguration[self.getPluginName()] = _dict
else:
self.__edConfiguration[self.getPluginName()] = {}
config = property(getConfig, setConfig)
def configure(self):
"""
Should be overridden by the Final Plugin If needed
This method should set its proper members attributes from a Plugin configuration Object
"""
self.DEBUG("EDPlugin.configure : plugin name = %s, EDNA_SITE = %s" % (self.getPluginName(), EDUtilsPath.EDNA_SITE))
# set Timeout if different from default one
if self.getTimeOut() == self.getDefaultTimeOut():
# Try to get time out from plugin configuration
iTimeOut = self.config.get(EDPlugin.CONF_TIME_OUT, None)
if iTimeOut is not None:
self.DEBUG("EDPlugin.configure: Setting time out to %d s from plugin configuration." % iTimeOut)
self.setTimeOut(iTimeOut)
else:
self.DEBUG("EDPlugin.configure: timeout already set before plugin is configured.")
# Base directory
strBaseDirectory = self.getBaseDirectory()
if (strBaseDirectory is None):
# Try to get base directory from plugin configuration
strBaseDirectory = self.config.get(EDPlugin.CONF_BASE_DIR_LABEL, None)
if(strBaseDirectory is None):
# Try to get working directory from environment variable
strBaseDirectory = os.environ.get("EDNA_BASE_DIRECTORY")
if (strBaseDirectory is None):
self.DEBUG("EDPlugin.configure: Using current base directory as working directory.")
strBaseDirectory = os.getcwd()
else:
self.DEBUG("EDPlugin.configure: Setting base directory from $EDNA_WORKING_DIRECTORY.")
else:
if (strBaseDirectory == "."):
self.DEBUG("EDPlugin.configure: Using current base directory as working directory.")
strBaseDirectory = os.getcwd()
else:
strBaseDirectory = os.path.abspath(strBaseDirectory)
self.DEBUG("EDPlugin.configure: Setting base directory from plugin configuration.")
self.setBaseDirectory(strBaseDirectory)
else:
self.DEBUG("EDPlugin.configure: Base directory already set before plugin is configured.")
# Working directory
strWorkingDirectory = self.getWorkingDirectory()
if (strWorkingDirectory is None):
# Try to get working directory from plugin configuration
strWorkingDirectory = self.config.get(EDPlugin.CONF_WORKING_DIR_LABEL, None)
if(strWorkingDirectory is not None):
self.DEBUG("EDPlugin.configure: Setting working directory from plugin configuration.")
else:
self.DEBUG("EDPlugin.configure: Setting working directory as base directory + base name.")
strWorkingDirectory = os.path.join(self.getBaseDirectory(), self.getBaseName())
self.setWorkingDirectory(strWorkingDirectory)
else:
self.DEBUG("EDPlugin.configure: Working directory already set before plugin is configured.")
#
self.__bWriteDataXMLInputOutput = bool(self.config.get(self.CONF_WRITE_XML_INPUT_OUTPUT, True))
self.__bWriteDataXMLOutput = bool(self.config.get(self.CONF_WRITE_XML_OUTPUT, True))
self.__bWriteDataXMLInput = bool(self.config.get(self.CONF_WRITE_XML_INPUT, True))
def execute(self, _edObject=None):
# Configure the plugin before starting it's thread
self.configure()
EDAction.execute(self, _edObject)
def checkParameters(self):
"""
Should be overridden by the Final Plugin If needed
This method should check that the data input are consistent
"""
self.DEBUG("EDPlugin.checkParameters")
def setXSDataInputClass(self, _xsDataInputClass, _strDataInputKey=None):
"""
This method should be called in the constructor of the derived plugins
in order to set the XSData type of the input data, e.g. XSDataInputXXX
"""
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInputClass.keys()):
strErrorMessage = "ERROR: " + self.getPluginName() + ".setXSDataInputClass, Data Input Class already defined for key: " + strDataInputKey
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError(strErrorMessage)
self.__dictXSDataInputClass[ strDataInputKey ] = _xsDataInputClass
def getXSDataInputClass(self, _strDataInputKey=None):
"""
Returns the XSData type of the input data.
"""
pyXSDataInputClass = None
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInputClass.keys()):
pyXSDataInputClass = self.__dictXSDataInputClass[ strDataInputKey ]
return pyXSDataInputClass
def setDataInput(self, _oDataInput, _strDataInputKey=None):
"""
Sets the plugin input data.
_oDataInput could be either an String XML or an XSData object.
The input data is stored in a dictionary with the key _strDataInputKey.
If the key is not provided a default key is used.
If not data input class is defined for the key an exception is raised.
If the key is not the default key, the data object is added to a list which
might contain already stored object(s).
If _oDataInput is None the list corresponding to a keyword is deleted.
"""
self.DEBUG("EDPlugin.setDataInput")
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
# Allow for None input
if (_oDataInput is None):
self.DEBUG("EDPlugin.setDataInput: Data input is None")
self.__dictXSDataInput[ strDataInputKey ] = []
elif (self.getXSDataInputClass(strDataInputKey) is None):
strErrorMessage = "ERROR: " + self.getPluginName() + ".setDataInput, Data Input Class not defined for key: " + strDataInputKey
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError(strErrorMessage)
else:
# Check the type
xsDataInput = None
if isinstance(_oDataInput, (str, unicode)):
self.DEBUG("EDPlugin.setDataInput: Input Data is string ")
xsDataInput = self.getXSDataInputClass(strDataInputKey).parseString(_oDataInput)
elif (isinstance(_oDataInput, self.getXSDataInputClass(strDataInputKey))):
self.DEBUG("EDPlugin.setDataInput: Input Data is of type " + str(_oDataInput.__class__))
xsDataInput = _oDataInput
else:
strErrorMessage = "ERROR: %s.setDataInput, wrong data type %r for data input key: %s, expected XML string or %r" % \
(self.getPluginName(), _oDataInput.__class__, strDataInputKey, self.getXSDataInputClass(strDataInputKey))
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError(strErrorMessage)
# Add the object to a list if its key is not the default key
if (strDataInputKey != self.__strDefaultInputDataKey) :
# Check if there's already a list stored
if (not strDataInputKey in self.__dictXSDataInput.keys()):
self.__dictXSDataInput[ strDataInputKey ] = []
self.__dictXSDataInput[ strDataInputKey ].append(xsDataInput)
else:
self.__dictXSDataInput[ strDataInputKey ] = xsDataInput
def hasDataInput(self, _strDataInputKey=None):
"""
Returns True if the plugin has Input Data for a particular key.
If the key is not provided a default key is used.
"""
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInput.keys()):
return True
else:
return False
def getDataInput(self, _strDataInputKey=None):
"""
Returns the Plugin Input Data for a particular key.
If the key is not provided a default key is used.
"""
oValue = None
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInput.keys()):
oValue = self.__dictXSDataInput[ strDataInputKey ]
else:
strErrorMessage = self.getPluginName() + ".getDataInput, no input data defined for key: " + strDataInputKey
self.warning(strErrorMessage)
self.addWarningMessage(strErrorMessage)
return oValue
def delDataInput(self, _strDataInputKey=None):
"""
Deletes the data input for a particular key.
If the key is not provided a default key is used.
"""
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInput.keys()):
self.__dictXSDataInput[ strDataInputKey ] = None
else:
strErrorMessage = self.getPluginName() + ".delDataInput, no input data defined for key: " + strDataInputKey
self.warning(strErrorMessage)
self.addWarningMessage(strErrorMessage)
# Property for dataInput
dataInput = property(getDataInput, setDataInput, delDataInput, "Property for dataInput")
def setDataOutput(self, _xsDataOutput, _strDataOutputKey=None):
"""
Sets the plugin output data for a particular key.
If the key is not provided a default key is used.
If the key is already defined in the dictionary, the corresponding
data object is added to a list which contains the already stored object(s).
"""
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
# Add the object to a list if its key not the default key
if (strDataOutputKey == self.__strDefaultOutputDataKey):
self.__dictXSDataOutput[ strDataOutputKey ] = _xsDataOutput
else:
# Check if the _xsDataoutput object is already a list
if (type(_xsDataOutput) == list):
self.__dictXSDataOutput[ strDataOutputKey ] = _xsDataOutput
else:
# Check if the stored object contains already a list
if (not strDataOutputKey in self.__dictXSDataOutput.keys()):
self.__dictXSDataOutput[ strDataOutputKey ] = []
self.__dictXSDataOutput[ strDataOutputKey ].append(_xsDataOutput)
def getDataOutput(self, _strDataOutputKey=None):
"""
Returns the Plugin Output Data
"""
oValue = None
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
if (strDataOutputKey in self.__dictXSDataOutput.keys()):
oValue = self.__dictXSDataOutput[ strDataOutputKey ]
return oValue
def hasDataOutput(self, _strDataOutputKey=None):
"""
Returns True if the plugin has the specified Output Data
"""
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
if (strDataOutputKey in self.__dictXSDataOutput.keys()):
return True
else:
return False
def delDataOutput(self, _strDataOutputKey=None):
"""
Deletes the data output for a particular key.
If the key is not provided a default key is used.
"""
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
if (strDataOutputKey in self.__dictXSDataOutput.keys()):
self.__dictXSDataOutput[ strDataOutputKey ] = None
else:
strErrorMessage = self.getPluginName() + ".delDataOutput, no output data defined for key: " + _strDataOutputKey
self.warning(strErrorMessage)
self.addWarningMessage(strErrorMessage)
# Property for dataOutput
dataOutput = property(getDataOutput, setDataOutput, delDataOutput, "Property for dataOutput")
def exportDataOutput(self, _edPlugin=None):
"""
Deprecated
Exports the Plugin Output Data to slot
"""
self.DEBUG("EDPlugin.exportDataOutput")
self.__edSlotExportDataOutput.call(self.__dictXSDataOutput)
def connectExportDataOutput(self, _oMethod):
"""
Deprecated
"""
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotExportDataOutput.connect(_oMethod)
self.synchronizeOff()
def generateExecutiveSummary(self, _edPlugin):
"""
This method, which should be implemented by sub-classes, generates an executive summary (user-related output summary).
"""
self.DEBUG("EDPlugin.generateExecutiveSummary")
def addErrorMessage(self, _strErrorMessage):
"""
Adds an error message to the error messages list
"""
self.__listErrorMessages.append(_strErrorMessage)
def getListOfErrorMessages(self):
"""
Returns the error messages list
"""
return self.__listErrorMessages
def addWarningMessage(self, _strWarningMessage):
"""
Adds a warning message to the warning messages list
"""
self.DEBUG("EDPlugin.addWarningMessage : " + _strWarningMessage)
self.__listWarningMessages.append(_strWarningMessage)
def getListOfWarningMessages(self):
"""
Returns the warning messages list
"""
return self.__listWarningMessages
def writeDataInput(self, _edObject=None):
"""
Writes the input data object(s) into a working dir xml file
"""
self.DEBUG("EDPlugin.writeDataInput")
strBasename = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()))
for strKey in self.__dictXSDataInput.keys():
if (strKey == self.__strDefaultInputDataKey): # "Old" style
xsDataInput = self.__dictXSDataInput[ self.__strDefaultInputDataKey ]
self.strPathDataInput = strBasename + "_dataInput.xml"
EDUtilsFile.writeFile(self.strPathDataInput, xsDataInput.marshal())
else: # We have a list of objects
listXSDataInput = self.__dictXSDataInput[ strKey ]
for iIndex, xsDataInput in enumerate(listXSDataInput):
strPathDataInput = "%s_%s_%d_dataInput.xml" % (strBasename, strKey, iIndex)
EDUtilsFile.writeFile(strPathDataInput, xsDataInput.marshal())
def writeDataOutput(self, _edObject=None):
"""
Writes the output data object(s) into a working dir xml file
"""
self.DEBUG("EDPlugin.writeDataOutput")
for strKey in self.__dictXSDataOutput.keys():
if (strKey == self.__strDefaultOutputDataKey): # "Old" style
xsDataOutput = self.__dictXSDataOutput[ self.__strDefaultOutputDataKey ]
if (xsDataOutput is not None):
self.strPathDataOutput = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()) + "_dataOutput.xml")
EDUtilsFile.writeFile(self.strPathDataOutput, xsDataOutput.marshal())
else:
listXSDataOutput = self.__dictXSDataOutput[ strKey ]
for iIndex, xsDataOutput in enumerate(listXSDataOutput):
if (xsDataOutput is not None):
strPathDataOutput = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()) + "_" + strKey + "_%d_dataOutput.xml" % iIndex)
EDUtilsFile.writeFile(strPathDataOutput, xsDataOutput.marshal())
def getBaseName(self):
"""
Returns the plugin base name
"""
if (self.__strBaseName is None):
self.__strBaseName = self.createBaseName()
return self.__strBaseName
def setBaseName(self, _strBaseName):
"""
Sets the plugin base name
"""
self.__strBaseName = self.compactPluginName(_strBaseName)
self.setName(self.__strBaseName)
# Create the directory baseDirectory/baseName which will be used as working directory
strWorkingDirPath = os.path.join(self.getBaseDirectory(), self.__strBaseName)
if not os.path.isdir(strWorkingDirPath):
os.mkdir(strWorkingDirPath)
self.setWorkingDirectory(strWorkingDirPath)
def createBaseName(self):
"""
Generates the plugin base name: (<prefix>-<object ID>)
"""
# First try to use global instance ID from EDObject
strBaseName = "%s-%08d" % (self.compactPluginName(self.getPluginName()), self.getId())
strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName)
# Try to create the directory...
try:
os.mkdir(strBaseDir)
except Exception as strErrorDetail:
self.error("EDPlugin.createBaseName: Could not create base directory %s because of %s" % (strBaseDir, strErrorDetail))
self.warning("EDPlugin.createBaseName: Trying to create alternative base directory...")
self.writeErrorTrace()
strTempDir = tempfile.mkdtemp(prefix=strBaseName, dir=self.getBaseDirectory())
os.chmod(strTempDir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
strBaseName = os.path.split(strTempDir)[1]
strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName)
self.warning("EDPlugin.createBaseName: Alternative base directory created: %s" % strBaseDir)
self.DEBUG("EDPlugin.createBaseName : Directory created = " + strBaseDir)
return strBaseName
def compactPluginName(self, _pluginName):
"""
The prefix is constructed from the plugin name with the following renaming:
EDPlugin -> ""
EDPluginExec -> ""
EDPluginControl -> "Control"
"""
strCompactName = _pluginName
if (strCompactName.startswith("EDPluginExec")):
strCompactName = strCompactName[12:]
elif (strCompactName.startswith("EDPluginControl")):
strCompactName = strCompactName[8:]
elif (strCompactName.startswith("EDPlugin")):
strCompactName = strCompactName[8:]
return strCompactName
def setBaseDirectory(self, _strBaseDirectory):
"""
Sets the plugin base directory
"""
self.DEBUG("EDPlugin.setBaseDirectory : " + _strBaseDirectory)
if (not os.path.isdir(_strBaseDirectory)):
self.DEBUG("EDPlugin.setBaseDirectory: base directory %s does dot yet exist! Creating it." % _strBaseDirectory)
os.mkdir(_strBaseDirectory)
self.__strBaseDirectory = _strBaseDirectory
def getBaseDirectory(self):
"""
Returns the plugin base directory
"""
self.DEBUG("EDPlugin.getBaseDirectory : %s" % self.__strBaseDirectory)
if (self.__strBaseDirectory is None):
self.__strBaseDirectory = os.getcwd()
return self.__strBaseDirectory
def setWorkingDirectory(self, _strWorkingDirectory):
"""
Sets the plugin working directory
"""
self.DEBUG("EDPlugin.setWorkingDirectory : " + _strWorkingDirectory)
self.__strWorkingDirectory = _strWorkingDirectory
if not os.path.isdir(_strWorkingDirectory):
self.DEBUG("EDPlugin.setWorkingDirectory, creating working directory %s." % _strWorkingDirectory)
os.mkdir(self.__strWorkingDirectory)
def getWorkingDirectory(self):
"""
Returns the plugin base directory
"""
self.DEBUG("EDPlugin.getWorkingDirectory : %s" % self.__strWorkingDirectory)
returnValue = None
if (self.__strWorkingDirectory is not None):
returnValue = self.__strWorkingDirectory
return returnValue
def checkMandatoryParameters(self, _xsData, _strParamName):
"""
Checks that a mandatory parameter exists in the data
If not, an error message is added in the list and the plugin fails
"""
if _xsData is None or (hasattr(_xsData, '__len__') and len(_xsData) == 0):
strErrorMessage = "%s: input parameter is missing: %s" % (self.getPluginName(), _strParamName)
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError(strErrorMessage)
def checkImportantParameters(self, _xsData, _strParamName):
"""
Checks that a specific parameter exists in the data
If not, a warning message is added in the list
"""
if(_xsData == None):
strWarningMessage = "%s: input parameter is missing: %s" % (self.getPluginName(), _strParamName)
self.warning(strWarningMessage)
self.addWarningMessage(strWarningMessage)
def addExecutiveSummaryLine(self, _strExecutiveSummaryLine):
"""
Add a line to the executive summary string.
"""
self.DEBUG("EDPlugin.addExecutiveSummaryLine : %r" % _strExecutiveSummaryLine)
strExecutiveSummaryLine = _strExecutiveSummaryLine
if (not strExecutiveSummaryLine == ""):
if (strExecutiveSummaryLine[-1] == "\n"):
strExecutiveSummaryLine = strExecutiveSummaryLine[:-1]
self.__listExecutiveSummaryLines.append(strExecutiveSummaryLine)
def addExecutiveSummarySeparator(self, _strSeparator=None):
"""
Adds a separator to split the executive summary into different parts
Default is a dotted line
"""
strSeparator = _strSeparator
if (strSeparator is None):
strSeparator = self.__strExecutiveSummarySeparator
# Check that the last line doesn't already contain a separator
if self.__listExecutiveSummaryLines != []:
if self.__listExecutiveSummaryLines[-1] != strSeparator:
self.addExecutiveSummaryLine(strSeparator)
else:
self.addExecutiveSummaryLine(strSeparator)
def getListExecutiveSummaryLines(self):
"""
Returns the executive summary (list of text lines)
"""
return self.__listExecutiveSummaryLines
def verboseScreenExecutiveSummary(self):
"""
Prints the executive summary on the screen
"""
for line in self.getListExecutiveSummaryLines():
self.screen(line)
def verboseDebug(self, _strMessage):
self.DEBUG(self.getPluginName() + " : " + _strMessage)
def getPluginName(self):
return self.getClassName()
def getListOfDataInputKeys(self):
return self.__dictXSDataInput.keys()
def getListOfDataOutputKeys(self):
return self.__dictXSDataOutput.keys()
def getDefaultInputDataKey(self):
return self.__strDefaultInputDataKey
def getDefaultOutputDataKey(self):
return self.__strDefaultOutputDataKey
def getExecutiveSummarySeparator(self):
return self.__strExecutiveSummarySeparator
def isRequiredToHaveConfiguration(self):
"""
If the return value from this method is true, the plugin
is required to have a configuration in order to be executed in a
plugin execution test case.
@return: RequiredToHaveConfiguration
@rtype: boolean
"""
return self.__isRequiredToHaveConfiguration
def setRequiredToHaveConfiguration(self, _bValue=True):
"""
Sets or unsets the plugin to be required to have a configuration
for execution in a plugin execution test case.
plugin execution test case.
@param _bValue: RequiredToHaveConfiguration
@type: boolean
"""
self.__isRequiredToHaveConfiguration = _bValue
def setWriteXMLInputOutput(self, _bValue=True):
"""
Sets or unsets the plugin to write XML input and output files.
@param _bValue: WriteDataXMLInputOutput
@type: boolean
"""
self.__bWriteDataXMLInputOutput = _bValue
def setWriteXMLInput(self, _bValue=True):
"""
Sets or unsets the plugin to write XML input files.
@param _bValue: WriteDataXMLInput
@type: boolean
"""
self.__bWriteDataXMLInput = _bValue
def setWriteXMLOutput(self, _bValue=True):
"""
Sets or unsets the plugin to write XML output files.
@param _bValue: WriteDataXMLOutput
@type: boolean
"""
self.__bWriteDataXMLOutput = _bValue
def setUseWarningInsteadOfError(self, _bValue=True):
"""
Sets or unsets the plugin to use warning messages also for error messages.
@param _bValue: UseWarningInsteadOfError
@type: boolean
"""
self.__bUseWarningInsteadOfError = _bValue
def error(self, _strErrorMessage):
"""
Overloaded from EDLogging. If self.__bUseWarningInsteadOfError is True
a warning message is issued instead of an error message.
"""
if self.__bUseWarningInsteadOfError:
self.warning(_strErrorMessage)
else:
EDAction.error(self, _strErrorMessage)
def ERROR(self, _strErrorMessage):
"""
Uses the overloaded self.error method above.
"""
self.error(_strErrorMessage)
class EDJob(EDLogging):
"""
Create a module called EDJob
* Most of what was done up to 09-2010 in EDParallelExecute should be done here
* Each instance will be a job
* Constructor takes a plugin name
* Each instance will have taking an "setDataInput" method getting an XMLin (as string)
* Each instance will gave a "getDataOutput" method with optional join
* there could be a "join" method, waiting for the job to finish
* Each instance will have a "execute" method and returning a JobId
* Each instance will have a "setCallBack" method that stores the name of the external callback
* provide status of a job
* keep track of all plugin status
* leave the time to plugin to initialize
* static class retrieve job-instance, status, small-log ...
* prevent multiple run of a single job ?
* does not manage workload of the computer, should be managed at the ExecPlugin level
Used for the tango binding, EDParallelExecute ...
== class variables ==
dictPluginStatus[pluginName] = ["uninitialized"|"running"|"executed"| "failed"]
dictJobs [JobId] = EDJob.Instance
== static methods ==
getJob(JobId)
"""
PLUGIN_STATE_UNITIALIZED = "uninitialized"
PLUGIN_STATE_RUNNING = "running"
PLUGIN_STATE_SUCCESS = "success"
PLUGIN_STATE_FAILURE = "failure"
__edFactoryPlugin = EDFactoryPlugin()
__dictJobs = {}
__semaphore = Semaphore()
__fStartTime = time.time()
def __init__(self, _strPluginName):
"""
Constructor of the class
@param strPluginName: name of the plugin
@type strPluginName: string
"""
EDLogging.__init__(self)
self.__strPluginName = _strPluginName
self.__edPlugin = None
self.__edSlotCallBack = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__pathXSDInput = None
self.__pathXSDOutput = None
self.__bXmlInputSet = False
self.__status = None
self.__name = None
self.__runtime = None
self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName)
if self.__edPlugin is None:
raise RuntimeError("Unable to create plugin %s" % self.__strPluginName)
self.__jobId = "%s-%08i" % (self.__strPluginName, self.__edPlugin.getId())
with self.__class__.__semaphore:
self.__class__.__dictJobs[self.__jobId] = self
if (self.__edPlugin is None):
self.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName)
else:
self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def setDataInput(self, _oDataInput, _strDataInputKey=None):
"""
Sets the job (plugin) input data.
@param: _oDataInput: could be either an String XML or an XSData object.
@param _strDataInputKey: the key of an input data dictionnary
The input data is stored in a dictionary with the key _strDataInputKey.
If the key is not provided a default key is used.
If not data input class is defined for the key an exception is raised.
If the key is not the default key, the data object is added to a list which
might contain already stored object(s).
If _oDataInput is None the list corresponding to a keyword is deleted.
"""
if _oDataInput in ["", None]:
self.__bXmlInputSet = False
return
else:
with self.locked():
if (self.__edPlugin is not None):
self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey)
self.__bXmlInputSet = True
else:
self.WARNING("Setting DataInput for uninstanciated plugin %s." % self.__strPluginName)
def getDataInput(self, _strDataInputKey=None):
"""
Returns the Plugin Input Data for a particular key.
If the key is not provided a default key is used.
"""
if (self.__edPlugin is not None):
return self.__edPlugin.getDataInput(_strDataInputKey).marshal()
elif (self.__pathXSDInput is not None):
return open(self.__pathXSDInput).read()
else:
self.WARNING("Getting DataInput for uninstanciated plugin %s." % self.__strPluginName)
dataInput = property(getDataInput, setDataInput)
def getDataOutput(self, _strDataOutputKey=None, _bWait=True):
"""
Returns the Plugin Output Data
@param _bWait: shall we wait for the plugin to finish to retrieve output data: Yes by default.
@type _bWait: boolean
"""
if _bWait: #Wait for plugin to finish befor returning data output
self.synchronize()
if (self.__edPlugin is not None):
return self.__edPlugin.getDataOutput(_strDataOutputKey).marshal()
elif self.__pathXSDOutput is not None:
return open(self.__pathXSDOutput).read()
else:
self.WARNING("Getting DataOutput for uninstanciated plugin or plugin has been garbage collected or output data were not saved. JobID was %s ." % self.__jobId)
dataOutput = property(getDataOutput)
def execute(self):
"""
Launch the EDNA plugin
@return: JobId
@rtype: string
"""
if not self.__bXmlInputSet:
self.WARNING("Not executing job %s as input is empty" % self.__jobId)
if (self.__edPlugin is not None):
with self.locked():
self.__edPlugin.connectSUCCESS(self.successPluginExecution)
self.__edPlugin.connectFAILURE(self.failurePluginExecution)
self.__status = EDJob.PLUGIN_STATE_RUNNING
self.__edPlugin.execute()
return self.__jobId
else:
self.WARNING("Trying to run a plugin that does not exist: %s " % self.__strPluginName)
def synchronize(self):
"""
Synchronize the execution of the job with the calling thread.
"""
with self.locked():
strStatus = self.__status
if strStatus == EDJob.PLUGIN_STATE_RUNNING:
self.__edPlugin.synchronize()
elif strStatus == EDJob.PLUGIN_STATE_UNITIALIZED:
self.WARNING("Unable to synchronize %s jobs" % strStatus)
else:
self.DEBUG("Unable to synchronize %s jobs" % strStatus)
@classmethod
def synchronizeAll(cls):
"""
Wait for all jobs to finish.
"""
EDVerbose.DEBUG("EDJob.synchronizeAll class method ")
listJob = cls.__dictJobs.keys()
for jobid in listJob:
job = cls.__dictJobs[jobid]
job.synchronize()
if len(cls.__dictJobs) != len(listJob):
EDVerbose.WARNING("EDJob.synchronizeAll: New jobs have been launched while synchronizing")
def successPluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin succeeds
"""
with self.locked():
self.__status = EDJob.PLUGIN_STATE_SUCCESS
self.screen("Plugin %s: success after %.3fs" % (self.__jobId, _edObject.getRunTime()))
try:
self.__edSlotSUCCESS.call(self.__jobId)
except Exception:
self.ERROR("Error in execution of Success call-back for %s" % self.__jobId)
self.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
self.ERROR("Error in execution of Common call-back (after success) for %s" % self.__jobId)
self.writeErrorTrace()
def failurePluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin failed
"""
with self.locked():
self.__status = EDJob.PLUGIN_STATE_FAILURE
self.screen("Plugin %s: failure after %.3fs" % (self.__jobId, _edObject.getRunTime()))
try:
self.__edSlotFAILURE.call(self.__jobId)
except Exception:
self.ERROR("Error in execution of Failure call-back for %s" % self.__jobId)
self.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
self.ERROR("Error in execution of Common call-back (after failure) for %s" % self.__jobId)
self.writeErrorTrace()
def connectSUCCESS(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotSUCCESS.connect(_oMethod)
def connectFAILURE(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotFAILURE.connect(_oMethod)
def connectCallBack(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotCallBack.connect(_oMethod)
def getJobId(self):
"""
@return: JobId i.e. EDPluginName-Number
@rtype: string
"""
return self.__jobId
jobId = property(getJobId, "EDJob.jobId: read-only property")
getJobID = getJobId
def getPluginName(self):
"""
@return: Name of the plugin
@rtype: string
"""
return self.__strPluginName
pluginName = property(getPluginName, "EDJob.pluginName: read-only property")
def getPlugin(self):
"""
@return: the plugin (instance)
@rtype: python object
"""
return self.__edPlugin
plugin = property(getPlugin, "EDJob.plugin: read-only property")
def getStatus(self):
"""
@return: status of the Job
@rtype: string
"""
return self.__status
status = property(getStatus, "EDJob.status: read-only property")
def getName(self):
return self.__name
def setName(self, _strName):
if self.__name is None:
self.__name = _strName
else:
self.WARNING("EDJob.setName: One cannot rename a Job !!!")
name = property(getName, setName, "EDJob.name: nickname of the job")
def getMemSize(self):
"""
try to guess the size in memory of a job
@return: expected size in memory
"""
if asizeof is not None:
return asizeof.asizeof(self)
@classmethod
def getStatusFromID(cls, jobId):
"""
Retrieve the job (hence the plugin) status
@param jobId: the Job identification number
@type jobId: string
@return: the EDJob status
@rtype: string
"""
if jobId in cls.__dictJobs:
strRet = cls.__dictJobs[jobId].getStatus()
else:
strRet = "Unable to retrieve such job: %s" % jobId
EDVerbose.WARNING(strRet)
return strRet
getStatusFromId = getStatusFromID
@classmethod
def getJobFromID(cls, jobId):
"""
Retrieve the job (hence the plugin)
@param jobId: the Job identification number
@type jobId: string
@return: the "EDJob instance", which contains the plugin (__edPlugin) and the status
@rtype: a Python object.
"""
if jobId in cls.__dictJobs:
return cls.__dictJobs[jobId]
else:
EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId)
getJobFromId = getJobFromID
@classmethod
def getMemoryFootprint(cls):
if asizeof is not None:
return asizeof.asizesof(cls.__dictJobs)
def cleanJob(self, forceGC=True):
"""
Frees the memory associated with the top level plugin
@param forceGC: Force garbage collection after clean-up
@type forceGC: boolean
"""
self.synchronize()
with self.locked():
if self.__edPlugin is not None:
self.__pathXSDOutput = self.__edPlugin.strPathDataOutput
self.__pathXSDInput = self.__edPlugin.strPathDataInput
self.__runtime = self.__edPlugin.getRunTime()
self.__edPlugin = None
if forceGC:
gc.collect()
@classmethod
def cleanJobfromId(cls, jobId, forceGC=True):
"""
Frees the memory associated with the top level plugin
@param jobId: the Job identification number
@type jobId: string
@param forceGC: Force garbage collection after clean-up
@type forceGC: boolean
"""
if jobId in cls.__dictJobs:
job = cls.__dictJobs[jobId]
job.cleanJob(forceGC)
strRet = "Job %s cleaned" % jobId
else:
strRet = "Unable to retrieve such EDJob: %s" % jobId
EDVerbose.WARNING(strRet)
return strRet
cleanJobfromID = cleanJobfromId
@classmethod
def getDataOutputFromId(cls, jobId):
"""
Returns the Plugin Output Data
@param jobId: job idenfier
@type jobId: string
@return: edJob.DataOutput XML string
"""
output = None
job = cls.getJobFromId(jobId)
if job is not None:
output = job.getDataOutput()
return output or ""
getDataOutputFromID = getDataOutputFromId
@classmethod
def getDataInputFromId(cls, jobId):
"""
Returns the Plugin Input Data
@param jobId: job idenfier
@type jobId: string
@return: edJob.DataInput XML string
"""
output = None
job = cls.getJobFromId(jobId)
if job is not None:
output = job.getDataInput()
return output or ""
getDataInputFromID = getDataInputFromId
@classmethod
def stats(cls):
"""
Retrieve some statistics and print them
"""
lstStrOut = []
output = []
fExecTime = time.time() - cls.__fStartTime
keys = cls.__dictJobs.keys()
keys.sort()
for num, key in enumerate(keys) :
job = cls.__dictJobs[key]
if job.getPlugin() is None:
runtime = job.__runtime
else:
runtime = job.getPlugin().getRunTime()
output.append([num, key, job.getStatus(), runtime, job.getMemSize()])
output.sort()
iNbJob = max(1, len(keys))
lstStrOut.append("_" * 110)
lstStrOut.append("%s\t|\t%s\t\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("nr", "EDPluginName-Id", "status", "runtime", "memory"))
lstStrOut.append("_" * 110)
fWall = 0.0
fSumProd = 0.0
fSumX = 0.0
fSumX2 = 0.0
for oneJob in output:
fWall += oneJob[3]
fSumX += oneJob[0]
fSumX2 += oneJob[0] * oneJob[0]
fSumProd += oneJob[0] * oneJob[3]
lstStrOut.append("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob))
lstStrOut.append("_" * 110)
lstStrOut.append("Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime))
lstStrOut.append("Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob))
if len(keys) > 1:
fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX)
fOrd = (fWall - fSlope * fSumX) / iNbJob
else:
fSlope = 0.0
fOrd = fWall
lstStrOut.append("Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope))
strOutput = os.linesep.join(lstStrOut)
EDVerbose.screen(strOutput)
return strOutput
class EDAction(EDLogging, Thread):
"""
This class is taking care of executing the EDNA plugin and
application workflow preProcess - process - postProcess.
The detailed workflow looks like this:
preProcess
if not failure: slotPreProcess
if not failure: process
if not failure: slotProcess
if not failure: postProcess
if not failure: slotPostProcess
if not failure: slotSUCCESS
if failure: slotFAILURE
Always: finally Process
"""
def __init__(self):
EDLogging.__init__(self)
Thread.__init__(self)
self.__edSlotPreProcess = EDSlot()
self.__edSlotProcess = EDSlot()
self.__edSlotPostProcess = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__edSlotFinallyProcess = EDSlot()
self.__bIsFailure = False
self.__bIsTimeOut = False
self.__fTimeOutInSeconds = None
self.__fDefaultTimeOutInSeconds = 600.0
self.__bIsAbort = False
# Reference to the object which calls execute or executeSynchronous
self.__edObject = None
self.__lExtraTime = [] # list of extra allowed time for execution (in second)
self.__bLogTiming = False
def executeKernel(self):
dictTimeStamps = { "init": time.time() }
self.setTimeInit()
try:
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel preProcess " + self.getClassName())
self.preProcess()
if self.__bLogTiming:
dictTimeStamps["preProcess"] = time.time()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel slotPreProcess " + self.getClassName())
self.__edSlotPreProcess.call(self)
if self.__bLogTiming:
dictTimeStamps["slotPreProcess"] = time.time()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel process " + self.getClassName())
self.process()
if self.__bLogTiming:
dictTimeStamps["process"] = time.time()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel slotProcess " + self.getClassName())
self.__edSlotProcess.call(self)
if self.__bLogTiming:
dictTimeStamps["slotProcess"] = time.time()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel postProcess " + self.getClassName())
self.postProcess()
if self.__bLogTiming:
dictTimeStamps["postProcess"] = time.time()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel slotPostProcess " + self.getClassName())
self.__edSlotPostProcess.call(self)
if self.__bLogTiming:
dictTimeStamps["slotPostProcess"] = time.time()
except Exception:
self.writeErrorTrace()
self.setFailure()
# Execute finally process even in case of failure
self.DEBUG("EDAction.executeKernel finallyProcess" + self.getClassName())
try:
self.finallyProcess()
if self.__bLogTiming:
dictTimeStamps["finallyProcess"] = time.time()
except Exception:
self.DEBUG("EDAction.executeKernel: ERROR in finallyProcess!")
self.writeErrorTrace()
self.setFailure()
try:
self.__edSlotFinallyProcess.call(self)
except Exception:
self.DEBUG("EDAction.executeKernel: ERROR in slotFinallyProcess!")
self.writeErrorTrace()
self.setFailure()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel slotSUCCESS")
# Check that something doesn't go wrong in the success method!
try:
self.__edSlotSUCCESS.call(self)
if self.__bLogTiming:
dictTimeStamps["slotSUCCESS"] = time.time()
except Exception:
self.DEBUG("EDAction.executeKernel: ERROR in slotSUCCESS!")
self.writeErrorTrace()
self.setFailure()
if (self.isFailure()):
self.DEBUG("EDAction.executeKernel slotFAILURE")
# Check that something doesn't go wrong in the success method!
try:
self.__edSlotFAILURE.call(self)
if self.__bLogTiming:
dictTimeStamps["slotFAILURE"] = time.time()
except Exception:
self.DEBUG("EDAction.executeKernel: ERROR in slotFAILURE!")
self.writeErrorTrace()
self.setTimeEnd()
if self.__bLogTiming:
lstTimings = []
dictTimeStamps["end"] = time.time()
lstTimings.append("EDAction.executeKernel: profiling of %s %i \t total time duration = %.3f s" % (self.getClassName(), self.getId(), dictTimeStamps["end"] - dictTimeStamps["init"]))
fTimeForFailureCalculation = dictTimeStamps["init"]
if "preProcess" in dictTimeStamps:
lstTimings.append("\t preProcess \t\t time duration = %.3f s" % (dictTimeStamps["preProcess"] - dictTimeStamps["init"]))
fTimeForFailureCalculation = dictTimeStamps["preProcess"]
if "process" in dictTimeStamps:
lstTimings.append("\t process \t\t time duration = %.3f s" % (dictTimeStamps["process"] - dictTimeStamps["slotPreProcess"]))
fTimeForFailureCalculation = dictTimeStamps["process"]
if "postProcess" in dictTimeStamps:
lstTimings.append("\t postProcess \t\t time duration = %.3f s" % (dictTimeStamps["postProcess"] - dictTimeStamps["slotProcess"]))
fTimeForFailureCalculation = dictTimeStamps["postProcess"]
fTimeForFinallyCalculation = fTimeForFailureCalculation
if "slotSUCCESS" in dictTimeStamps:
lstTimings.append("\t slotSUCCESS \t\t time duration = %.3f s" % (dictTimeStamps["slotSUCCESS"] - dictTimeStamps["slotPostProcess"]))
fTimeForFailureCalculation = dictTimeStamps["slotSUCCESS"]
if "slotFAILURE" in dictTimeStamps:
lstTimings.append("\t slotFAILURE \t\t time duration = %.3f s" % (dictTimeStamps["slotFAILURE"] - fTimeForFailureCalculation))
if dictTimeStamps.has_key("finallyProcess"):
lstTimings.append("\t finallyProcess \t time duration = %.3f s" % (dictTimeStamps["finallyProcess"] - fTimeForFinallyCalculation))
self.log(os.linesep.join(lstTimings))
def synchronize(self):
"""
Wait for the thread to finish. Since the time out is used by
e.g. EDPluginExecProcessScript we add an extra second in order
to allow the subclasses to handle the time out - without this
extra second it's the EDAction class who time-outs first.
Note that this does not in any case add time to the execution,
because the extra second is only used for time-outs. The method
returns immediately once the thread has finished.
"""
EDVerbose.DEBUG("EDAction.synchronize() for %s" % self.getName())
# fTimeOut = self.__fTimeOutInSeconds
if self.__fTimeOutInSeconds is None:
self.__fTimeOutInSeconds = self.__fDefaultTimeOutInSeconds
#Wait for the thread to be started up to timeout
if self.isStarted() == False:
tStartWait = time.time()
while self.isStarted() == False:
time.sleep(1)
if time.time() - tStartWait > self.__fTimeOutInSeconds:
self.__bIsTimeOut = True
strErrorMessage = "Timeout when waiting for %s to start!" % self.getClassName()
EDVerbose.DEBUG("EDAction.synchronize: " + strErrorMessage)
EDVerbose.ERROR(strErrorMessage)
self.setFailure()
return
# We add an extra second in order to allow execution plugin to finish
# which have the same timeout
self.join(float(self.__fTimeOutInSeconds + 1))
for fExtraTime in self.__lExtraTime:
self.join(float(fExtraTime))
if self.isAlive():
# Timeout!
self.__bIsTimeOut = True
EDVerbose.DEBUG("EDAction.synchronize: Timeout!")
strErrorMessage = "Timeout when waiting for %s to terminate." % self.getClassName()
EDVerbose.ERROR(strErrorMessage)
self.setFailure()
def isTimeOut(self):
return self.__bIsTimeOut
def hasTimedOut(self, _bTimeout):
"""
Enforce the timeout state
@param _bTimeout: if you think you can force this !
@type _bTimeout: boolean
"""
self.__bIsTimeOut = bool(_bTimeout)
def isFailure(self):
return self.__bIsFailure
def setFailure(self):
self.__bIsFailure = True
def run(self):
self.executeKernel()
def executeAction(self, _edObject=None):
self.execute(_edObject)
def executeActionSynchronous(self, _edObject=None):
self.executeSynchronous()
def connectPreProcess(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotPreProcess.connect(_oMethod)
self.synchronizeOff()
def connectProcess(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotProcess.connect(_oMethod)
self.synchronizeOff()
def connectPostProcess(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotPostProcess.connect(_oMethod)
self.synchronizeOff()
def connectSUCCESS(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotSUCCESS.connect(_oMethod)
self.synchronizeOff()
def connectFAILURE(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotFAILURE.connect(_oMethod)
self.synchronizeOff()
def connectFinallyProcess(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotFinallyProcess.connect(_oMethod)
self.synchronizeOff()
def isRunning(self):
return self.isAlive()
def isEnded(self):
EDVerbose.DEBUG("%s.isEnded return %s" % (self.getName(), (self.getTimeEnd() is not None)))
return (self.getTimeEnd() is not None)
def isStarted(self):
EDVerbose.DEBUG("%s.isStarted return %s, %s " % (self.getName(), (self.getTimeInit() is not None), self.getTimeInit()))
return (self.getTimeInit() is not None)
def setTimeOut(self, _fTimeOut):
"""
Sets the time out
"""
EDVerbose.DEBUG("EDAction.setTimeOut called with value %s" % _fTimeOut)
self.__fTimeOutInSeconds = float(_fTimeOut)
def getTimeOut(self):
"""
Returns the time out
"""
if self.__fTimeOutInSeconds is None:
self.__fTimeOutInSeconds = self.__fDefaultTimeOutInSeconds
return self.__fTimeOutInSeconds
def addExtraTime(self, _fExtraTime):
"""
Allows to incread the timeout of a plugin while it is running
@param _fExtraTime: extra time to be added to timeout
@type _fExtraTime: float
"""
self.__lExtraTime.append(_fExtraTime)
self.setTimeOut(self.getTimeOut() + _fExtraTime)
def getDefaultTimeOut(self):
"""
Returns the time out
"""
return self.__fDefaultTimeOutInSeconds
def getSlotSUCCESS(self):
return self.__edSlotSUCCESS
def getSlotFAILURE(self):
return self.__edSlotFAILURE
def preProcess(self, _edObject=None):
pass
def process(self, _edObject=None):
pass
def postProcess(self, _edObject=None):
pass
def abort(self, _edObject=None):
pass
def finallyProcess(self, _edObject=None):
pass
def execute(self, _edObject=None):
self.__bIsStarted = True
self.__edObject = _edObject
self.start()
def executeSynchronous(self, _edObject=None):
self.execute(_edObject)
self.synchronize()
def setLogTiming(self, _bValue):
"""
Force this action to log it's timing to file
"""
self.__bLogTiming = bool(_bValue)
def getLogTiming(self):
return self.__bLogTiming
logTiming = property(getLogTiming, setLogTiming)
class EDAction(EDLogging, Thread):
"""
This class is taking care of executing the EDNA plugin and
application workflow preProcess - process - postProcess.
The detailed workflow looks like this:
preProcess
if not failure: slotPreProcess
if not failure: process
if not failure: slotProcess
if not failure: postProcess
if not failure: slotPostProcess
if not failure: slotSUCCESS
if failure: slotFAILURE
Always: finally Process
"""
def __init__(self):
EDLogging.__init__(self)
Thread.__init__(self)
self.__edSlotPreProcess = EDSlot()
self.__edSlotProcess = EDSlot()
self.__edSlotPostProcess = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__edSlotFinallyProcess = EDSlot()
self.__bIsFailure = False
self.__bIsTimeOut = False
self.__fTimeOutInSeconds = None
self.__fDefaultTimeOutInSeconds = 600.0
self.__bIsAbort = False
# Reference to the object which calls execute or executeSynchronous
self.__edObject = None
self.__lExtraTime = [
] # list of extra allowed time for execution (in second)
self.__bLogTiming = False
def executeKernel(self):
dictTimeStamps = {"init": time.time()}
self.setTimeInit()
try:
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel preProcess " +
self.getClassName())
self.preProcess()
if self.__bLogTiming:
dictTimeStamps["preProcess"] = time.time()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel slotPreProcess " +
self.getClassName())
self.__edSlotPreProcess.call(self)
if self.__bLogTiming:
dictTimeStamps["slotPreProcess"] = time.time()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel process " +
self.getClassName())
self.process()
if self.__bLogTiming:
dictTimeStamps["process"] = time.time()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel slotProcess " +
self.getClassName())
self.__edSlotProcess.call(self)
if self.__bLogTiming:
dictTimeStamps["slotProcess"] = time.time()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel postProcess " +
self.getClassName())
self.postProcess()
if self.__bLogTiming:
dictTimeStamps["postProcess"] = time.time()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel slotPostProcess " +
self.getClassName())
self.__edSlotPostProcess.call(self)
if self.__bLogTiming:
dictTimeStamps["slotPostProcess"] = time.time()
except Exception:
self.writeErrorTrace()
self.setFailure()
# Execute finally process even in case of failure
self.DEBUG("EDAction.executeKernel finallyProcess" +
self.getClassName())
try:
self.finallyProcess()
if self.__bLogTiming:
dictTimeStamps["finallyProcess"] = time.time()
except Exception:
self.DEBUG("EDAction.executeKernel: ERROR in finallyProcess!")
self.writeErrorTrace()
self.setFailure()
try:
self.__edSlotFinallyProcess.call(self)
except Exception:
self.DEBUG("EDAction.executeKernel: ERROR in slotFinallyProcess!")
self.writeErrorTrace()
self.setFailure()
if (not self.isFailure()):
self.DEBUG("EDAction.executeKernel slotSUCCESS")
# Check that something doesn't go wrong in the success method!
try:
self.__edSlotSUCCESS.call(self)
if self.__bLogTiming:
dictTimeStamps["slotSUCCESS"] = time.time()
except Exception:
self.DEBUG("EDAction.executeKernel: ERROR in slotSUCCESS!")
self.writeErrorTrace()
self.setFailure()
if (self.isFailure()):
self.DEBUG("EDAction.executeKernel slotFAILURE")
# Check that something doesn't go wrong in the success method!
try:
self.__edSlotFAILURE.call(self)
if self.__bLogTiming:
dictTimeStamps["slotFAILURE"] = time.time()
except Exception:
self.DEBUG("EDAction.executeKernel: ERROR in slotFAILURE!")
self.writeErrorTrace()
self.setTimeEnd()
if self.__bLogTiming:
lstTimings = []
dictTimeStamps["end"] = time.time()
lstTimings.append(
"EDAction.executeKernel: profiling of %s %i \t total time duration = %.3f s"
% (self.getClassName(), self.getId(),
dictTimeStamps["end"] - dictTimeStamps["init"]))
fTimeForFailureCalculation = dictTimeStamps["init"]
if "preProcess" in dictTimeStamps:
lstTimings.append(
"\t preProcess \t\t time duration = %.3f s" %
(dictTimeStamps["preProcess"] - dictTimeStamps["init"]))
fTimeForFailureCalculation = dictTimeStamps["preProcess"]
if "process" in dictTimeStamps:
lstTimings.append("\t process \t\t time duration = %.3f s" %
(dictTimeStamps["process"] -
dictTimeStamps["slotPreProcess"]))
fTimeForFailureCalculation = dictTimeStamps["process"]
if "postProcess" in dictTimeStamps:
lstTimings.append(
"\t postProcess \t\t time duration = %.3f s" %
(dictTimeStamps["postProcess"] -
dictTimeStamps["slotProcess"]))
fTimeForFailureCalculation = dictTimeStamps["postProcess"]
fTimeForFinallyCalculation = fTimeForFailureCalculation
if "slotSUCCESS" in dictTimeStamps:
lstTimings.append(
"\t slotSUCCESS \t\t time duration = %.3f s" %
(dictTimeStamps["slotSUCCESS"] -
dictTimeStamps["slotPostProcess"]))
fTimeForFailureCalculation = dictTimeStamps["slotSUCCESS"]
if "slotFAILURE" in dictTimeStamps:
lstTimings.append(
"\t slotFAILURE \t\t time duration = %.3f s" %
(dictTimeStamps["slotFAILURE"] -
fTimeForFailureCalculation))
if "finallyProcess" in dictTimeStamps:
lstTimings.append(
"\t finallyProcess \t time duration = %.3f s" %
(dictTimeStamps["finallyProcess"] -
fTimeForFinallyCalculation))
self.log(os.linesep.join(lstTimings))
def synchronize(self):
"""
Wait for the thread to finish. Since the time out is used by
e.g. EDPluginExecProcessScript we add an extra second in order
to allow the subclasses to handle the time out - without this
extra second it's the EDAction class who time-outs first.
Note that this does not in any case add time to the execution,
because the extra second is only used for time-outs. The method
returns immediately once the thread has finished.
"""
EDVerbose.DEBUG("EDAction.synchronize() for %s" % self.getName())
# fTimeOut = self.__fTimeOutInSeconds
if self.__fTimeOutInSeconds is None:
self.__fTimeOutInSeconds = self.__fDefaultTimeOutInSeconds
#Wait for the thread to be started up to timeout
if self.isStarted() == False:
tStartWait = time.time()
while self.isStarted() == False:
time.sleep(1)
if time.time() - tStartWait > self.__fTimeOutInSeconds:
self.__bIsTimeOut = True
strErrorMessage = "Timeout when waiting for %s to start!" % self.getClassName(
)
EDVerbose.DEBUG("EDAction.synchronize: " + strErrorMessage)
EDVerbose.ERROR(strErrorMessage)
self.setFailure()
return
# We add an extra second in order to allow execution plugin to finish
# which have the same timeout
self.join(float(self.__fTimeOutInSeconds + 1))
for fExtraTime in self.__lExtraTime:
self.join(float(fExtraTime))
if self.isAlive():
# Timeout!
self.__bIsTimeOut = True
EDVerbose.DEBUG("EDAction.synchronize: Timeout!")
strErrorMessage = "Timeout when waiting for %s to terminate." % self.getClassName(
)
EDVerbose.ERROR(strErrorMessage)
self.setFailure()
def isTimeOut(self):
return self.__bIsTimeOut
def hasTimedOut(self, _bTimeout):
"""
Enforce the timeout state
@param _bTimeout: if you think you can force this !
@type _bTimeout: boolean
"""
self.__bIsTimeOut = bool(_bTimeout)
def isFailure(self):
return self.__bIsFailure
def setFailure(self, value=True):
"""
use setFailure(False) to unset the failure flag
"""
self.__bIsFailure = value
def run(self):
self.executeKernel()
def executeAction(self, _edObject=None):
self.execute(_edObject)
def executeActionSynchronous(self, _edObject=None):
self.executeSynchronous()
def connectPreProcess(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotPreProcess.connect(_oMethod)
self.synchronizeOff()
def connectProcess(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotProcess.connect(_oMethod)
self.synchronizeOff()
def connectPostProcess(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotPostProcess.connect(_oMethod)
self.synchronizeOff()
def connectSUCCESS(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotSUCCESS.connect(_oMethod)
self.synchronizeOff()
def connectFAILURE(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotFAILURE.connect(_oMethod)
self.synchronizeOff()
def connectFinallyProcess(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotFinallyProcess.connect(_oMethod)
self.synchronizeOff()
def isRunning(self):
return self.isAlive()
def isEnded(self):
EDVerbose.DEBUG("%s.isEnded return %s" %
(self.getName(), (self.getTimeEnd() is not None)))
return (self.getTimeEnd() is not None)
def isStarted(self):
EDVerbose.DEBUG("%s.isStarted return %s, %s " %
(self.getName(),
(self.getTimeInit() is not None), self.getTimeInit()))
return (self.getTimeInit() is not None)
def setTimeOut(self, _fTimeOut):
"""
Sets the time out
"""
EDVerbose.DEBUG("EDAction.setTimeOut called with value %s" % _fTimeOut)
self.__fTimeOutInSeconds = float(_fTimeOut)
def getTimeOut(self):
"""
Returns the time out
"""
if self.__fTimeOutInSeconds is None:
self.__fTimeOutInSeconds = self.__fDefaultTimeOutInSeconds
return self.__fTimeOutInSeconds
def addExtraTime(self, _fExtraTime):
"""
Allows to incread the timeout of a plugin while it is running
@param _fExtraTime: extra time to be added to timeout
@type _fExtraTime: float
"""
self.__lExtraTime.append(_fExtraTime)
self.setTimeOut(self.getTimeOut() + _fExtraTime)
def getDefaultTimeOut(self):
"""
Returns the time out
"""
return self.__fDefaultTimeOutInSeconds
def getSlotSUCCESS(self):
return self.__edSlotSUCCESS
def getSlotFAILURE(self):
return self.__edSlotFAILURE
def preProcess(self, _edObject=None):
pass
def process(self, _edObject=None):
pass
def postProcess(self, _edObject=None):
pass
def abort(self, _edObject=None):
pass
def finallyProcess(self, _edObject=None):
pass
def execute(self, _edObject=None):
self.__bIsStarted = True
self.__edObject = _edObject
self.start()
def executeSynchronous(self, _edObject=None):
self.execute(_edObject)
self.synchronize()
def setLogTiming(self, _bValue):
"""
Force this action to log it's timing to file
"""
self.__bLogTiming = bool(_bValue)
def getLogTiming(self):
return self.__bLogTiming
logTiming = property(getLogTiming, setLogTiming)
class EDPlugin(EDAction):
"""
This is the EDNA plugin main class
An EDNA plugin class:
- is a configurable entity
- has a base name (<date>-<random number>-<base name>)
- handles input/output data (setter, getter, checker)
- has warning and error messages
- has a base and a working directory (both are configurable)
The working directory is the folder from which the plugin is launched
and should contain all associated files with the plugin execution (edna xml input/output, 3rd party output files)
The base directory is the parent directory of the working directory.
Example: the working directory of a control plugin is the base directory of the plugins that it invokes,
i.e. the plugins working directories has the control plugin working directory as parent.
- defines the method that generates an executive summary (user-related output summary) that sub-classes should implement
"""
CONF_BASE_DIR_LABEL = "baseDirectory"
CONF_WORKING_DIR_LABEL = "workingDirectory"
CONF_TIME_OUT = "timeOut"
CONF_WRITE_XML_INPUT_OUTPUT = "writeXMLInputOutput"
CONF_WRITE_XML_OUTPUT = "writeXMLOutput"
CONF_WRITE_XML_INPUT = "writeXMLInput"
def __init__(self):
"""
Initializes plugin related attributes described above
"""
EDAction.__init__(self)
self.__xsPluginItem = None
self.__dictXSDataInputClass = {}
self.__dictXSDataInput = {}
self.__dictXSDataOutput = {}
self.__strDefaultInputDataKey = "defaultInputData"
self.__strDefaultOutputDataKey = "defaultOutputData"
self.__edSlotExportDataOutput = EDSlot()
self.__strBaseDirectory = None
self.__strWorkingDirectory = None
self.__strBaseName = None
self.__listExecutiveSummaryLines = []
self.__strExecutiveSummarySeparator = "-" * 80
self.__listErrorMessages = []
self.__listWarningMessages = []
self.__isRequiredToHaveConfiguration = False
self.__bWriteDataXMLInputOutput = True
self.__bWriteDataXMLOutput = True
self.__bWriteDataXMLInput = True
self.__strPluginId = "%s-%08i" % (self.getClassName(), self.getId())
self.strPathDataInput = None
self.strPathDataOutput = None
self.__bUseWarningInsteadOfError = False
self.__edConfiguration = EDConfigurationStatic()
def preProcess(self, _edObject=None):
"""
Writes xml data input in the working dir (if required)
Connects a slot for generating the executive summary after the plugin execution
Connects a slot for checking output data to the finally process
Initialize the base directory
Configures the plugin
Checks the input data
"""
EDAction.preProcess(self, _edObject)
self.DEBUG("EDPlugin.preProcess")
self.connectPostProcess(self.exportDataOutput)
if self.__bWriteDataXMLInputOutput:
if self.__bWriteDataXMLInput:
self.connectPreProcess(self.writeDataInput)
self.connectPostProcess(self.generateExecutiveSummary)
self.connectFinallyProcess(self.checkDataOutput)
if (self.__strBaseName is None):
self.setBaseName(self.createBaseName())
EDStatus.tellRunning(self.__strPluginId)
self.connectFinallyProcess(self.tellFinished)
self.checkParameters()
def tellFinished(self, _edObject=None):
"""
Tell EDStatus that the plugin has finished, either in success either in error
"""
if self.isFailure():
EDStatus.tellFailure(self.__strPluginId)
else:
EDStatus.tellSuccess(self.__strPluginId)
def checkDataOutput(self, _edObject=None):
"""
Checks if output data is available, if not issues a warning and sets an empty XSDataResult as output data
Writes xml data output in the working dir (if required)
"""
EDAction.finallyProcess(self, _edObject)
if self.__dictXSDataOutput == {}:
strWarningMessage = "Output data for plugin %s not set, using XSDataResult as output" % self.getPluginName(
)
self.WARNING(strWarningMessage)
self.addWarningMessage(strWarningMessage)
self.setDataOutput(XSDataResult())
if self.__bWriteDataXMLInputOutput:
if self.__bWriteDataXMLOutput:
self.writeDataOutput()
def synchronize(self):
"""
This method calls EDAction.synchronize and if a time-out occurs an error message
is added to the list of error messages.
"""
EDAction.synchronize(self)
if self.isTimeOut():
strErrorMessage = "Timeout when waiting for %s to terminate." % self.getClassName(
)
self.addErrorMessage(strErrorMessage)
def getConfig(self):
"""
Gets the Plugin Configuration as a dictionary
"""
# self.DEBUG("EDPlugin.getConfig")
return self.__edConfiguration.get(self.getPluginName(), {})
def setConfig(self, _dict, _bLocal=False):
"""
Receives a dictionary (Plugin Configuration) from the application.
"""
self.DEBUG("EDPlugin.setConfiguration")
if _bLocal:
self.__edConfiguration = EDConfiguration()
if _dict is not None:
self.__edConfiguration[self.getPluginName()] = _dict
else:
self.__edConfiguration[self.getPluginName()] = {}
config = property(getConfig, setConfig)
def configure(self):
"""
Should be overridden by the Final Plugin If needed
This method should set its proper members attributes from a Plugin configuration Object
"""
self.DEBUG("EDPlugin.configure : plugin name = %s, EDNA_SITE = %s" %
(self.getPluginName(), EDUtilsPath.EDNA_SITE))
# set Timeout if different from default one
if self.getTimeOut() == self.getDefaultTimeOut():
# Try to get time out from plugin configuration
iTimeOut = self.config.get(EDPlugin.CONF_TIME_OUT, None)
if iTimeOut is not None:
self.DEBUG(
"EDPlugin.configure: Setting time out to %d s from plugin configuration."
% iTimeOut)
self.setTimeOut(iTimeOut)
else:
self.DEBUG(
"EDPlugin.configure: timeout already set before plugin is configured."
)
# Base directory
strBaseDirectory = self.getBaseDirectory()
if (strBaseDirectory is None):
# Try to get base directory from plugin configuration
strBaseDirectory = self.config.get(EDPlugin.CONF_BASE_DIR_LABEL,
None)
if (strBaseDirectory is None):
# Try to get working directory from environment variable
strBaseDirectory = os.environ.get("EDNA_BASE_DIRECTORY")
if (strBaseDirectory is None):
self.DEBUG(
"EDPlugin.configure: Using current base directory as working directory."
)
strBaseDirectory = os.getcwd()
else:
self.DEBUG(
"EDPlugin.configure: Setting base directory from $EDNA_WORKING_DIRECTORY."
)
else:
if (strBaseDirectory == "."):
self.DEBUG(
"EDPlugin.configure: Using current base directory as working directory."
)
strBaseDirectory = os.getcwd()
else:
strBaseDirectory = os.path.abspath(strBaseDirectory)
self.DEBUG(
"EDPlugin.configure: Setting base directory from plugin configuration."
)
self.setBaseDirectory(strBaseDirectory)
else:
self.DEBUG(
"EDPlugin.configure: Base directory already set before plugin is configured."
)
# Working directory
strWorkingDirectory = self.getWorkingDirectory()
if (strWorkingDirectory is None):
# Try to get working directory from plugin configuration
strWorkingDirectory = self.config.get(
EDPlugin.CONF_WORKING_DIR_LABEL, None)
if (strWorkingDirectory is not None):
self.DEBUG(
"EDPlugin.configure: Setting working directory from plugin configuration."
)
else:
self.DEBUG(
"EDPlugin.configure: Setting working directory as base directory + base name."
)
strWorkingDirectory = os.path.join(self.getBaseDirectory(),
self.getBaseName())
self.setWorkingDirectory(strWorkingDirectory)
else:
self.DEBUG(
"EDPlugin.configure: Working directory already set before plugin is configured."
)
#
self.__bWriteDataXMLInputOutput = bool(
self.config.get(self.CONF_WRITE_XML_INPUT_OUTPUT, True))
self.__bWriteDataXMLOutput = bool(
self.config.get(self.CONF_WRITE_XML_OUTPUT, True))
self.__bWriteDataXMLInput = bool(
self.config.get(self.CONF_WRITE_XML_INPUT, True))
def execute(self, _edObject=None):
# Configure the plugin before starting it's thread
self.configure()
EDAction.execute(self, _edObject)
def checkParameters(self):
"""
Should be overridden by the Final Plugin If needed
This method should check that the data input are consistent
"""
self.DEBUG("EDPlugin.checkParameters")
def setXSDataInputClass(self, _xsDataInputClass, _strDataInputKey=None):
"""
This method should be called in the constructor of the derived plugins
in order to set the XSData type of the input data, e.g. XSDataInputXXX
"""
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInputClass.keys()):
strErrorMessage = "ERROR: " + self.getPluginName(
) + ".setXSDataInputClass, Data Input Class already defined for key: " + strDataInputKey
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError(strErrorMessage)
self.__dictXSDataInputClass[strDataInputKey] = _xsDataInputClass
def getXSDataInputClass(self, _strDataInputKey=None):
"""
Returns the XSData type of the input data.
"""
pyXSDataInputClass = None
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInputClass.keys()):
pyXSDataInputClass = self.__dictXSDataInputClass[strDataInputKey]
return pyXSDataInputClass
def setDataInput(self, _oDataInput, _strDataInputKey=None):
"""
Sets the plugin input data.
_oDataInput could be either an String XML or an XSData object.
The input data is stored in a dictionary with the key _strDataInputKey.
If the key is not provided a default key is used.
If not data input class is defined for the key an exception is raised.
If the key is not the default key, the data object is added to a list which
might contain already stored object(s).
If _oDataInput is None the list corresponding to a keyword is deleted.
"""
self.DEBUG("EDPlugin.setDataInput")
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
# Allow for None input
if (_oDataInput is None):
self.DEBUG("EDPlugin.setDataInput: Data input is None")
self.__dictXSDataInput[strDataInputKey] = []
elif (self.getXSDataInputClass(strDataInputKey) is None):
strErrorMessage = "ERROR: " + self.getPluginName(
) + ".setDataInput, Data Input Class not defined for key: " + strDataInputKey
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError(strErrorMessage)
else:
# Check the type
xsDataInput = None
if isinstance(_oDataInput, (str, unicode)):
self.DEBUG("EDPlugin.setDataInput: Input Data is string ")
xsDataInput = self.getXSDataInputClass(
strDataInputKey).parseString(_oDataInput)
elif (isinstance(_oDataInput,
self.getXSDataInputClass(strDataInputKey))):
self.DEBUG("EDPlugin.setDataInput: Input Data is of type " +
str(_oDataInput.__class__))
xsDataInput = _oDataInput
else:
strErrorMessage = "ERROR: %s.setDataInput, wrong data type %r for data input key: %s, expected XML string or %r" % \
(self.getPluginName(), _oDataInput.__class__, strDataInputKey, self.getXSDataInputClass(strDataInputKey))
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError(strErrorMessage)
# Add the object to a list if its key is not the default key
if (strDataInputKey != self.__strDefaultInputDataKey):
# Check if there's already a list stored
if (not strDataInputKey in self.__dictXSDataInput.keys()):
self.__dictXSDataInput[strDataInputKey] = []
self.__dictXSDataInput[strDataInputKey].append(xsDataInput)
else:
self.__dictXSDataInput[strDataInputKey] = xsDataInput
def hasDataInput(self, _strDataInputKey=None):
"""
Returns True if the plugin has Input Data for a particular key.
If the key is not provided a default key is used.
"""
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInput.keys()):
return True
else:
return False
def getDataInput(self, _strDataInputKey=None):
"""
Returns the Plugin Input Data for a particular key.
If the key is not provided a default key is used.
"""
oValue = None
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInput.keys()):
oValue = self.__dictXSDataInput[strDataInputKey]
else:
strErrorMessage = self.getPluginName(
) + ".getDataInput, no input data defined for key: " + strDataInputKey
self.warning(strErrorMessage)
self.addWarningMessage(strErrorMessage)
return oValue
def delDataInput(self, _strDataInputKey=None):
"""
Deletes the data input for a particular key.
If the key is not provided a default key is used.
"""
strDataInputKey = _strDataInputKey
if (strDataInputKey is None):
strDataInputKey = self.__strDefaultInputDataKey
if (strDataInputKey in self.__dictXSDataInput.keys()):
self.__dictXSDataInput[strDataInputKey] = None
else:
strErrorMessage = self.getPluginName(
) + ".delDataInput, no input data defined for key: " + strDataInputKey
self.warning(strErrorMessage)
self.addWarningMessage(strErrorMessage)
# Property for dataInput
dataInput = property(getDataInput, setDataInput, delDataInput,
"Property for dataInput")
def setDataOutput(self, _xsDataOutput, _strDataOutputKey=None):
"""
Sets the plugin output data for a particular key.
If the key is not provided a default key is used.
If the key is already defined in the dictionary, the corresponding
data object is added to a list which contains the already stored object(s).
"""
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
# Add the object to a list if its key not the default key
if (strDataOutputKey == self.__strDefaultOutputDataKey):
self.__dictXSDataOutput[strDataOutputKey] = _xsDataOutput
else:
# Check if the _xsDataoutput object is already a list
if (type(_xsDataOutput) == list):
self.__dictXSDataOutput[strDataOutputKey] = _xsDataOutput
else:
# Check if the stored object contains already a list
if (not strDataOutputKey in self.__dictXSDataOutput.keys()):
self.__dictXSDataOutput[strDataOutputKey] = []
self.__dictXSDataOutput[strDataOutputKey].append(_xsDataOutput)
def getDataOutput(self, _strDataOutputKey=None):
"""
Returns the Plugin Output Data
"""
oValue = None
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
if (strDataOutputKey in self.__dictXSDataOutput.keys()):
oValue = self.__dictXSDataOutput[strDataOutputKey]
return oValue
def hasDataOutput(self, _strDataOutputKey=None):
"""
Returns True if the plugin has the specified Output Data
"""
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
if (strDataOutputKey in self.__dictXSDataOutput.keys()):
return True
else:
return False
def delDataOutput(self, _strDataOutputKey=None):
"""
Deletes the data output for a particular key.
If the key is not provided a default key is used.
"""
strDataOutputKey = _strDataOutputKey
if (strDataOutputKey is None):
strDataOutputKey = self.__strDefaultOutputDataKey
if (strDataOutputKey in self.__dictXSDataOutput.keys()):
self.__dictXSDataOutput[strDataOutputKey] = None
else:
strErrorMessage = self.getPluginName(
) + ".delDataOutput, no output data defined for key: " + _strDataOutputKey
self.warning(strErrorMessage)
self.addWarningMessage(strErrorMessage)
# Property for dataOutput
dataOutput = property(getDataOutput, setDataOutput, delDataOutput,
"Property for dataOutput")
def exportDataOutput(self, _edPlugin=None):
"""
Deprecated
Exports the Plugin Output Data to slot
"""
self.DEBUG("EDPlugin.exportDataOutput")
self.__edSlotExportDataOutput.call(self.__dictXSDataOutput)
def connectExportDataOutput(self, _oMethod):
"""
Deprecated
"""
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotExportDataOutput.connect(_oMethod)
self.synchronizeOff()
def generateExecutiveSummary(self, _edPlugin):
"""
This method, which should be implemented by sub-classes, generates an executive summary (user-related output summary).
"""
self.DEBUG("EDPlugin.generateExecutiveSummary")
def addErrorMessage(self, _strErrorMessage):
"""
Adds an error message to the error messages list
"""
self.__listErrorMessages.append(_strErrorMessage)
def getListOfErrorMessages(self):
"""
Returns the error messages list
"""
return self.__listErrorMessages
def addWarningMessage(self, _strWarningMessage):
"""
Adds a warning message to the warning messages list
"""
self.DEBUG("EDPlugin.addWarningMessage : " + _strWarningMessage)
self.__listWarningMessages.append(_strWarningMessage)
def getListOfWarningMessages(self):
"""
Returns the warning messages list
"""
return self.__listWarningMessages
def writeDataInput(self, _edObject=None):
"""
Writes the input data object(s) into a working dir xml file
"""
self.DEBUG("EDPlugin.writeDataInput")
strBasename = os.path.join(
self.getWorkingDirectory(),
self.compactPluginName(self.getPluginName()))
for strKey in self.__dictXSDataInput.keys():
if (strKey == self.__strDefaultInputDataKey): # "Old" style
xsDataInput = self.__dictXSDataInput[
self.__strDefaultInputDataKey]
self.strPathDataInput = strBasename + "_dataInput.xml"
EDUtilsFile.writeFile(self.strPathDataInput,
xsDataInput.marshal())
else: # We have a list of objects
listXSDataInput = self.__dictXSDataInput[strKey]
for iIndex, xsDataInput in enumerate(listXSDataInput):
strPathDataInput = "%s_%s_%d_dataInput.xml" % (
strBasename, strKey, iIndex)
EDUtilsFile.writeFile(strPathDataInput,
xsDataInput.marshal())
def writeDataOutput(self, _edObject=None):
"""
Writes the output data object(s) into a working dir xml file
"""
self.DEBUG("EDPlugin.writeDataOutput")
for strKey in self.__dictXSDataOutput.keys():
if (strKey == self.__strDefaultOutputDataKey): # "Old" style
xsDataOutput = self.__dictXSDataOutput[
self.__strDefaultOutputDataKey]
if (xsDataOutput is not None):
self.strPathDataOutput = os.path.join(
self.getWorkingDirectory(),
self.compactPluginName(self.getPluginName()) +
"_dataOutput.xml")
EDUtilsFile.writeFile(self.strPathDataOutput,
xsDataOutput.marshal())
else:
listXSDataOutput = self.__dictXSDataOutput[strKey]
for iIndex, xsDataOutput in enumerate(listXSDataOutput):
if (xsDataOutput is not None):
strPathDataOutput = os.path.join(
self.getWorkingDirectory(),
self.compactPluginName(self.getPluginName()) +
"_" + strKey + "_%d_dataOutput.xml" % iIndex)
EDUtilsFile.writeFile(strPathDataOutput,
xsDataOutput.marshal())
def getBaseName(self):
"""
Returns the plugin base name
"""
if (self.__strBaseName is None):
self.__strBaseName = self.createBaseName()
return self.__strBaseName
def setBaseName(self, _strBaseName):
"""
Sets the plugin base name
"""
self.__strBaseName = self.compactPluginName(_strBaseName)
self.setName(self.__strBaseName)
# Create the directory baseDirectory/baseName which will be used as working directory
strWorkingDirPath = os.path.join(self.getBaseDirectory(),
self.__strBaseName)
if not os.path.isdir(strWorkingDirPath):
os.mkdir(strWorkingDirPath)
self.setWorkingDirectory(strWorkingDirPath)
def createBaseName(self):
"""
Generates the plugin base name: (<prefix>-<object ID>)
"""
# First try to use global instance ID from EDObject
strBaseName = "%s-%08d" % (self.compactPluginName(
self.getPluginName()), self.getId())
strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName)
# Try to create the directory...
try:
os.mkdir(strBaseDir)
except Exception as strErrorDetail:
self.error(
"EDPlugin.createBaseName: Could not create base directory %s because of %s"
% (strBaseDir, strErrorDetail))
self.warning(
"EDPlugin.createBaseName: Trying to create alternative base directory..."
)
self.writeErrorTrace()
strTempDir = tempfile.mkdtemp(prefix=strBaseName,
dir=self.getBaseDirectory())
os.chmod(
strTempDir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
| stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
strBaseName = os.path.split(strTempDir)[1]
strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName)
self.warning(
"EDPlugin.createBaseName: Alternative base directory created: %s"
% strBaseDir)
self.DEBUG("EDPlugin.createBaseName : Directory created = " +
strBaseDir)
return strBaseName
def compactPluginName(self, _pluginName):
"""
The prefix is constructed from the plugin name with the following renaming:
EDPlugin -> ""
EDPluginExec -> ""
EDPluginControl -> "Control"
"""
strCompactName = _pluginName
if (strCompactName.startswith("EDPluginExec")):
strCompactName = strCompactName[12:]
elif (strCompactName.startswith("EDPluginControl")):
strCompactName = strCompactName[8:]
elif (strCompactName.startswith("EDPlugin")):
strCompactName = strCompactName[8:]
return strCompactName
def setBaseDirectory(self, _strBaseDirectory):
"""
Sets the plugin base directory
"""
self.DEBUG("EDPlugin.setBaseDirectory : " + _strBaseDirectory)
if (not os.path.isdir(_strBaseDirectory)):
self.DEBUG(
"EDPlugin.setBaseDirectory: base directory %s does dot yet exist! Creating it."
% _strBaseDirectory)
os.mkdir(_strBaseDirectory)
self.__strBaseDirectory = _strBaseDirectory
def getBaseDirectory(self):
"""
Returns the plugin base directory
"""
self.DEBUG("EDPlugin.getBaseDirectory : %s" % self.__strBaseDirectory)
if (self.__strBaseDirectory is None):
self.__strBaseDirectory = os.getcwd()
return self.__strBaseDirectory
def setWorkingDirectory(self, _strWorkingDirectory):
"""
Sets the plugin working directory
"""
self.DEBUG("EDPlugin.setWorkingDirectory : " + _strWorkingDirectory)
self.__strWorkingDirectory = _strWorkingDirectory
if not os.path.isdir(_strWorkingDirectory):
self.DEBUG(
"EDPlugin.setWorkingDirectory, creating working directory %s."
% _strWorkingDirectory)
os.mkdir(self.__strWorkingDirectory)
def getWorkingDirectory(self):
"""
Returns the plugin base directory
"""
self.DEBUG("EDPlugin.getWorkingDirectory : %s" %
self.__strWorkingDirectory)
returnValue = None
if (self.__strWorkingDirectory is not None):
returnValue = self.__strWorkingDirectory
return returnValue
def checkMandatoryParameters(self, _xsData, _strParamName):
"""
Checks that a mandatory parameter exists in the data
If not, an error message is added in the list and the plugin fails
"""
if _xsData is None or (hasattr(_xsData, '__len__')
and len(_xsData) == 0):
strErrorMessage = "%s: input parameter is missing: %s" % (
self.getPluginName(), _strParamName)
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError(strErrorMessage)
def checkImportantParameters(self, _xsData, _strParamName):
"""
Checks that a specific parameter exists in the data
If not, a warning message is added in the list
"""
if (_xsData == None):
strWarningMessage = "%s: input parameter is missing: %s" % (
self.getPluginName(), _strParamName)
self.warning(strWarningMessage)
self.addWarningMessage(strWarningMessage)
def addExecutiveSummaryLine(self, _strExecutiveSummaryLine):
"""
Add a line to the executive summary string.
"""
self.DEBUG("EDPlugin.addExecutiveSummaryLine : %r" %
_strExecutiveSummaryLine)
strExecutiveSummaryLine = _strExecutiveSummaryLine
if (not strExecutiveSummaryLine == ""):
if (strExecutiveSummaryLine[-1] == "\n"):
strExecutiveSummaryLine = strExecutiveSummaryLine[:-1]
self.__listExecutiveSummaryLines.append(strExecutiveSummaryLine)
def addExecutiveSummarySeparator(self, _strSeparator=None):
"""
Adds a separator to split the executive summary into different parts
Default is a dotted line
"""
strSeparator = _strSeparator
if (strSeparator is None):
strSeparator = self.__strExecutiveSummarySeparator
# Check that the last line doesn't already contain a separator
if self.__listExecutiveSummaryLines != []:
if self.__listExecutiveSummaryLines[-1] != strSeparator:
self.addExecutiveSummaryLine(strSeparator)
else:
self.addExecutiveSummaryLine(strSeparator)
def getListExecutiveSummaryLines(self):
"""
Returns the executive summary (list of text lines)
"""
return self.__listExecutiveSummaryLines
def verboseScreenExecutiveSummary(self):
"""
Prints the executive summary on the screen
"""
for line in self.getListExecutiveSummaryLines():
self.screen(line)
def verboseDebug(self, _strMessage):
self.DEBUG(self.getPluginName() + " : " + _strMessage)
def getPluginName(self):
return self.getClassName()
def getListOfDataInputKeys(self):
return self.__dictXSDataInput.keys()
def getListOfDataOutputKeys(self):
return self.__dictXSDataOutput.keys()
def getDefaultInputDataKey(self):
return self.__strDefaultInputDataKey
def getDefaultOutputDataKey(self):
return self.__strDefaultOutputDataKey
def getExecutiveSummarySeparator(self):
return self.__strExecutiveSummarySeparator
def isRequiredToHaveConfiguration(self):
"""
If the return value from this method is true, the plugin
is required to have a configuration in order to be executed in a
plugin execution test case.
@return: RequiredToHaveConfiguration
@rtype: boolean
"""
return self.__isRequiredToHaveConfiguration
def setRequiredToHaveConfiguration(self, _bValue=True):
"""
Sets or unsets the plugin to be required to have a configuration
for execution in a plugin execution test case.
plugin execution test case.
@param _bValue: RequiredToHaveConfiguration
@type: boolean
"""
self.__isRequiredToHaveConfiguration = _bValue
def setWriteXMLInputOutput(self, _bValue=True):
"""
Sets or unsets the plugin to write XML input and output files.
@param _bValue: WriteDataXMLInputOutput
@type: boolean
"""
self.__bWriteDataXMLInputOutput = _bValue
def setWriteXMLInput(self, _bValue=True):
"""
Sets or unsets the plugin to write XML input files.
@param _bValue: WriteDataXMLInput
@type: boolean
"""
self.__bWriteDataXMLInput = _bValue
def setWriteXMLOutput(self, _bValue=True):
"""
Sets or unsets the plugin to write XML output files.
@param _bValue: WriteDataXMLOutput
@type: boolean
"""
self.__bWriteDataXMLOutput = _bValue
def setUseWarningInsteadOfError(self, _bValue=True):
"""
Sets or unsets the plugin to use warning messages also for error messages.
@param _bValue: UseWarningInsteadOfError
@type: boolean
"""
self.__bUseWarningInsteadOfError = _bValue
def error(self, _strErrorMessage):
"""
Overloaded from EDLogging. If self.__bUseWarningInsteadOfError is True
a warning message is issued instead of an error message.
"""
if self.__bUseWarningInsteadOfError:
self.warning(_strErrorMessage)
else:
EDAction.error(self, _strErrorMessage)
def ERROR(self, _strErrorMessage):
"""
Uses the overloaded self.error method above.
"""
self.error(_strErrorMessage)
class EDJob(EDLogging):
"""
Create a module called EDJob
* Most of what was done up to 09-2010 in EDParallelExecute should be done here
* Each instance will be a job
* Constructor takes a plugin name
* Each instance will have taking an "setDataInput" method getting an XMLin (as string)
* Each instance will gave a "getDataOutput" method with optional join
* there could be a "join" method, waiting for the job to finish
* Each instance will have a "execute" method and returning a JobId
* Each instance will have a "setCallBack" method that stores the name of the external callback
* provide status of a job
* keep track of all plugin status
* leave the time to plugin to initialize
* static class retrieve job-instance, status, small-log ...
* prevent multiple run of a single job ?
* does not manage workload of the computer, should be managed at the ExecPlugin level
Used for the tango binding, EDParallelExecute ...
== class variables ==
dictPluginStatus[pluginName] = ["uninitialized"|"running"|"executed"| "failed"]
dictJobs [JobId] = EDJob.Instance
== static methods ==
getJob(JobId)
"""
PLUGIN_STATE_UNITIALIZED = "uninitialized"
PLUGIN_STATE_RUNNING = "running"
PLUGIN_STATE_SUCCESS = "success"
PLUGIN_STATE_FAILURE = "failure"
__edFactoryPlugin = EDFactoryPlugin()
__dictJobs = {}
__semaphore = Semaphore()
__fStartTime = time.time()
def __init__(self, _strPluginName):
"""
Constructor of the class
@param strPluginName: name of the plugin
@type strPluginName: string
"""
EDLogging.__init__(self)
self.__strPluginName = _strPluginName
self.__edPlugin = None
self.__edSlotCallBack = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__pathXSDInput = None
self.__pathXSDOutput = None
self.__bXmlInputSet = False
self.__status = None
self.__name = None
self.__runtime = None
self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(
self.__strPluginName)
if self.__edPlugin is None:
raise RuntimeError("Unable to create plugin %s" %
self.__strPluginName)
self.__jobId = "%s-%08i" % (self.__strPluginName,
self.__edPlugin.getId())
with self.__class__.__semaphore:
self.__class__.__dictJobs[self.__jobId] = self
if (self.__edPlugin is None):
self.WARNING("Instantiation of plugin %s failed!!!" %
_strPluginName)
else:
self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def setDataInput(self, _oDataInput, _strDataInputKey=None):
"""
Sets the job (plugin) input data.
@param: _oDataInput: could be either an String XML or an XSData object.
@param _strDataInputKey: the key of an input data dictionnary
The input data is stored in a dictionary with the key _strDataInputKey.
If the key is not provided a default key is used.
If not data input class is defined for the key an exception is raised.
If the key is not the default key, the data object is added to a list which
might contain already stored object(s).
If _oDataInput is None the list corresponding to a keyword is deleted.
"""
if _oDataInput in ["", None]:
self.__bXmlInputSet = False
return
else:
with self.locked():
if (self.__edPlugin is not None):
self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey)
self.__bXmlInputSet = True
else:
self.WARNING(
"Setting DataInput for uninstanciated plugin %s." %
self.__strPluginName)
def getDataInput(self, _strDataInputKey=None):
"""
Returns the Plugin Input Data for a particular key.
If the key is not provided a default key is used.
"""
if (self.__edPlugin is not None):
return self.__edPlugin.getDataInput(_strDataInputKey).marshal()
elif (self.__pathXSDInput is not None):
return open(self.__pathXSDInput).read()
else:
self.WARNING("Getting DataInput for uninstanciated plugin %s." %
self.__strPluginName)
dataInput = property(getDataInput, setDataInput)
def getDataOutput(self, _strDataOutputKey=None, _bWait=True):
"""
Returns the Plugin Output Data
@param _bWait: shall we wait for the plugin to finish to retrieve output data: Yes by default.
@type _bWait: boolean
"""
if _bWait: # Wait for plugin to finish befor returning data output
self.synchronize()
if (self.__edPlugin is not None):
return self.__edPlugin.getDataOutput(_strDataOutputKey).marshal()
elif self.__pathXSDOutput is not None:
return open(self.__pathXSDOutput).read()
else:
self.WARNING(
"Getting DataOutput for uninstanciated plugin or plugin has been garbage collected or output data were not saved. JobID was %s ."
% self.__jobId)
dataOutput = property(getDataOutput)
def execute(self):
"""
Launch the EDNA plugin
@return: JobId
@rtype: string
"""
if not self.__bXmlInputSet:
self.WARNING("Not executing job %s as input is empty" %
self.__jobId)
if (self.__edPlugin is not None):
with self.locked():
self.__edPlugin.connectSUCCESS(self.successPluginExecution)
self.__edPlugin.connectFAILURE(self.failurePluginExecution)
self.__status = EDJob.PLUGIN_STATE_RUNNING
self.__edPlugin.execute()
return self.__jobId
else:
self.WARNING("Trying to run a plugin that does not exist: %s " %
self.__strPluginName)
def synchronize(self):
"""
Synchronize the execution of the job with the calling thread.
"""
with self.locked():
strStatus = self.__status
if strStatus == EDJob.PLUGIN_STATE_RUNNING:
self.__edPlugin.synchronize()
elif strStatus == EDJob.PLUGIN_STATE_UNITIALIZED:
self.WARNING("Unable to synchronize %s jobs" % strStatus)
else:
self.DEBUG("Unable to synchronize %s jobs" % strStatus)
@classmethod
def synchronizeAll(cls):
"""
Wait for all jobs to finish.
"""
EDVerbose.DEBUG("EDJob.synchronizeAll class method ")
listJob = cls.__dictJobs.keys()
for jobid in listJob:
job = cls.__dictJobs[jobid]
job.synchronize()
if len(cls.__dictJobs) != len(listJob):
EDVerbose.WARNING(
"EDJob.synchronizeAll: New jobs have been launched while synchronizing"
)
def successPluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin succeeds
"""
with self.locked():
self.__status = EDJob.PLUGIN_STATE_SUCCESS
self.screen("Plugin %s: success after %.3fs" %
(self.__jobId, _edObject.getRunTime()))
try:
self.__edSlotSUCCESS.call(self.__jobId)
except Exception:
self.ERROR("Error in execution of Success call-back for %s" %
self.__jobId)
self.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
self.ERROR(
"Error in execution of Common call-back (after success) for %s"
% self.__jobId)
self.writeErrorTrace()
def failurePluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin failed
"""
with self.locked():
self.__status = EDJob.PLUGIN_STATE_FAILURE
self.screen("Plugin %s: failure after %.3fs" %
(self.__jobId, _edObject.getRunTime()))
try:
self.__edSlotFAILURE.call(self.__jobId)
except Exception:
self.ERROR("Error in execution of Failure call-back for %s" %
self.__jobId)
self.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
self.ERROR(
"Error in execution of Common call-back (after failure) for %s"
% self.__jobId)
self.writeErrorTrace()
def connectSUCCESS(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotSUCCESS.connect(_oMethod)
def connectFAILURE(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotFAILURE.connect(_oMethod)
def connectCallBack(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotCallBack.connect(_oMethod)
def getJobId(self):
"""
@return: JobId i.e. EDPluginName-Number
@rtype: string
"""
return self.__jobId
jobId = property(getJobId, "EDJob.jobId: read-only property")
getJobID = getJobId
def getPluginName(self):
"""
@return: Name of the plugin
@rtype: string
"""
return self.__strPluginName
pluginName = property(getPluginName,
"EDJob.pluginName: read-only property")
def getPlugin(self):
"""
@return: the plugin (instance)
@rtype: python object
"""
return self.__edPlugin
plugin = property(getPlugin, "EDJob.plugin: read-only property")
def getStatus(self):
"""
@return: status of the Job
@rtype: string
"""
return self.__status
status = property(getStatus, "EDJob.status: read-only property")
def getName(self):
return self.__name
def setName(self, _strName):
if self.__name is None:
self.__name = _strName
else:
self.WARNING("EDJob.setName: One cannot rename a Job !!!")
name = property(getName, setName, "EDJob.name: nickname of the job")
def getMemSize(self):
"""
try to guess the size in memory of a job
@return: expected size in memory
"""
if asizeof is not None:
return asizeof.asizeof(self)
@classmethod
def getStatusFromID(cls, jobId):
"""
Retrieve the job (hence the plugin) status
@param jobId: the Job identification number
@type jobId: string
@return: the EDJob status
@rtype: string
"""
if jobId in cls.__dictJobs:
strRet = cls.__dictJobs[jobId].getStatus()
else:
strRet = "Unable to retrieve such job: %s" % jobId
EDVerbose.WARNING(strRet)
return strRet
getStatusFromId = getStatusFromID
@classmethod
def getJobFromID(cls, jobId):
"""
Retrieve the job (hence the plugin)
@param jobId: the Job identification number
@type jobId: string
@return: the "EDJob instance", which contains the plugin (__edPlugin) and the status
@rtype: a Python object.
"""
if jobId in cls.__dictJobs:
return cls.__dictJobs[jobId]
else:
EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId)
getJobFromId = getJobFromID
@classmethod
def getMemoryFootprint(cls):
if asizeof is not None:
return asizeof.asizesof(cls.__dictJobs)
def cleanJob(self, forceGC=True):
"""
Frees the memory associated with the top level plugin
@param forceGC: Force garbage collection after clean-up
@type forceGC: boolean
"""
self.synchronize()
with self.locked():
if self.__edPlugin is not None:
self.__pathXSDOutput = self.__edPlugin.strPathDataOutput
self.__pathXSDInput = self.__edPlugin.strPathDataInput
self.__runtime = self.__edPlugin.getRunTime()
self.__edPlugin = None
if forceGC:
gc.collect()
@classmethod
def cleanJobfromId(cls, jobId, forceGC=True):
"""
Frees the memory associated with the top level plugin
@param jobId: the Job identification number
@type jobId: string
@param forceGC: Force garbage collection after clean-up
@type forceGC: boolean
"""
if jobId in cls.__dictJobs:
job = cls.__dictJobs[jobId]
job.cleanJob(forceGC)
strRet = "Job %s cleaned" % jobId
else:
strRet = "Unable to retrieve such EDJob: %s" % jobId
EDVerbose.WARNING(strRet)
return strRet
cleanJobfromID = cleanJobfromId
@classmethod
def getDataOutputFromId(cls, jobId):
"""
Returns the Plugin Output Data
@param jobId: job idenfier
@type jobId: string
@return: edJob.DataOutput XML string
"""
output = None
job = cls.getJobFromId(jobId)
if job is not None:
output = job.getDataOutput()
return output or ""
getDataOutputFromID = getDataOutputFromId
@classmethod
def getDataInputFromId(cls, jobId):
"""
Returns the Plugin Input Data
@param jobId: job idenfier
@type jobId: string
@return: edJob.DataInput XML string
"""
output = None
job = cls.getJobFromId(jobId)
if job is not None:
output = job.getDataInput()
return output or ""
getDataInputFromID = getDataInputFromId
@classmethod
def countRunning(cls):
"""
return the number of jobs still running.
"""
running = 0
for jobid in list(cls.__dictJobs.keys()): # python3 !
running += (
cls.__dictJobs[jobid].__status == cls.PLUGIN_STATE_RUNNING)
return running
@classmethod
def stats(cls):
"""
Retrieve some statistics and print them
"""
lstStrOut = []
output = []
fExecTime = time.time() - cls.__fStartTime
keys = cls.__dictJobs.keys()
keys.sort()
for num, key in enumerate(keys):
job = cls.__dictJobs[key]
if job.getPlugin() is None:
runtime = job.__runtime
else:
runtime = job.getPlugin().getRunTime()
output.append(
[num, key,
job.getStatus(), runtime,
job.getMemSize()])
output.sort()
iNbJob = max(1, len(keys))
lstStrOut.append("_" * 110)
lstStrOut.append(
"%s\t|\t%s\t\t\t\t|\t%s\t|\t%s\t\t|\t%s" %
("nr", "EDPluginName-Id", "status", "runtime", "memory"))
lstStrOut.append("_" * 110)
fWall = 0.0
fSumProd = 0.0
fSumX = 0.0
fSumX2 = 0.0
for oneJob in output:
fWall += oneJob[3]
fSumX += oneJob[0]
fSumX2 += oneJob[0] * oneJob[0]
fSumProd += oneJob[0] * oneJob[3]
lstStrOut.append("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" %
tuple(oneJob))
lstStrOut.append("_" * 110)
lstStrOut.append(
"Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f"
% (fWall, fExecTime, fWall / fExecTime))
lstStrOut.append(
"Average execution time (Wall/N): %.3fs, Average throughput: %.3fs"
% (fWall / iNbJob, fExecTime / iNbJob))
if len(keys) > 1:
fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 -
fSumX * fSumX)
fOrd = (fWall - fSlope * fSumX) / iNbJob
else:
fSlope = 0.0
fOrd = fWall
lstStrOut.append(
"Regression of execution time: ExecTime = %.3f + %f * NbJob" %
(fOrd, fSlope))
strOutput = os.linesep.join(lstStrOut)
EDVerbose.screen(strOutput)
return strOutput
def __init__(self):
EDLogging.__init__(self)
Thread.__init__(self)
self.__edSlotPreProcess = EDSlot()
self.__edSlotProcess = EDSlot()
self.__edSlotPostProcess = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__edSlotFinallyProcess = EDSlot()
self.__bIsFailure = False
self.__bIsTimeOut = False
self.__fTimeOutInSeconds = None
self.__fDefaultTimeOutInSeconds = 600.0
self.__bIsAbort = False
# Reference to the object which calls execute or executeSynchronous
self.__edObject = None
self.__lExtraTime = [
] # list of extra allowed time for execution (in second)
self.__bLogTiming = False
