def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDLogging.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__pathXSDInput = None self.__pathXSDOutput = None self.__bXmlInputSet = False self.__status = None self.__name = None self.__runtime = None self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName) if self.__edPlugin is None: raise RuntimeError("Unable to create plugin %s" % self.__strPluginName) self.__jobId = "%s-%08i" % (self.__strPluginName, self.__edPlugin.getId()) with self.__class__.__semaphore: self.__class__.__dictJobs[self.__jobId] = self if (self.__edPlugin is None): self.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDObject.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__bXmlInputSet = False self.__status = None # self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName) self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName) EDJob.__semaphore.acquire() # Create the JobID if not self.__strPluginName in EDJob.__dictPluginLastId: EDJob.__dictPluginLastId[_strPluginName] = 0 else: EDJob.__dictPluginLastId[_strPluginName] += 1 self.__jobId = "%s-%i" % (self.__strPluginName, EDJob.__dictPluginLastId[_strPluginName]) EDJob.__dictJobs[self.__jobId] = self EDJob.__semaphore.release() if (self.__edPlugin is None): EDVerbose.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def __init__ (self): """ Initializes plugin related attributes described above """ EDAction.__init__(self) self.__xsPluginItem = None self.__dictXSDataInputClass = {} self.__dictXSDataInput = {} self.__dictXSDataOutput = {} self.__strDefaultInputDataKey = "defaultInputData" self.__strDefaultOutputDataKey = "defaultOutputData" self.__edSlotExportDataOutput = EDSlot() self.__strBaseDirectory = None self.__strWorkingDirectory = None self.__strBaseName = None self.__listExecutiveSummaryLines = [] self.__strExecutiveSummarySeparator = "-" * 80 self.__listErrorMessages = [] self.__listWarningMessages = [] self.__isRequiredToHaveConfiguration = False self.__bWriteDataXMLInputOutput = True self.__bWriteDataXMLOutput = True self.__bWriteDataXMLInput = True self.__strPluginId = "%s-%08i" % (self.getClassName(), self.getId()) self.strPathDataInput = None self.strPathDataOutput = None
def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDLogging.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__pathXSDInput = None self.__pathXSDOutput = None self.__bXmlInputSet = False self.__status = None self.__name = None self.__runtime = None self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin( self.__strPluginName) if self.__edPlugin is None: raise RuntimeError("Unable to create plugin %s" % self.__strPluginName) self.__jobId = "%s-%08i" % (self.__strPluginName, self.__edPlugin.getId()) with self.__class__.__semaphore: self.__class__.__dictJobs[self.__jobId] = self if (self.__edPlugin is None): self.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def executePluginSynchronous(self, _edPlugin): """ This method is used to start executable plugins in pipeline synchronously. """ _edControlSlotSUCCESS = EDSlot() _edControlSlotFAILURE = EDSlot() map(_edControlSlotSUCCESS.connect, _edPlugin.getSlotSUCCESS().getListMethod()) map(_edControlSlotFAILURE.connect, _edPlugin.getSlotFAILURE().getListMethod()) _edPlugin.getSlotSUCCESS().emptyListMethod() _edPlugin.getSlotFAILURE().emptyListMethod() _edPlugin.executeSynchronous() if (not _edPlugin.isFailure()): EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotSUCCESS") # Check that something doesn't go wrong in the success method! try: _edControlSlotSUCCESS.call(_edPlugin) except Exception: EDVerbose.DEBUG("EDControlPlugin.executeSynchronous: ERROR in slotSUCCESS!") EDVerbose.writeErrorTrace() _edPlugin.setFailure() if (_edPlugin.isFailure()): EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotFAILURE") # Check that something doesn't go wrong in the success method! try: _edControlSlotFAILURE.call(_edPlugin) except Exception: EDVerbose.DEBUG("EDControlPlugin.executeSynchronous: ERROR in slotFAILURE!") EDVerbose.writeErrorTrace()
def executePluginSynchronous(self, _edPlugin): """ This method is used to start executable plugins in pipeline synchronously. """ _edControlSlotSUCCESS = EDSlot() _edControlSlotFAILURE = EDSlot() map(_edControlSlotSUCCESS.connect, _edPlugin.getSlotSUCCESS().getListMethod()) map(_edControlSlotFAILURE.connect, _edPlugin.getSlotFAILURE().getListMethod()) _edPlugin.getSlotSUCCESS().emptyListMethod() _edPlugin.getSlotFAILURE().emptyListMethod() _edPlugin.executeSynchronous() if (not _edPlugin.isFailure()): EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotSUCCESS") # Check that something doesn't go wrong in the success method! try: _edControlSlotSUCCESS.call(_edPlugin) except Exception: EDVerbose.DEBUG( "EDControlPlugin.executeSynchronous: ERROR in slotSUCCESS!" ) EDVerbose.writeErrorTrace() _edPlugin.setFailure() if (_edPlugin.isFailure()): EDVerbose.DEBUG("EDControlPlugin.executeSynchronous slotFAILURE") # Check that something doesn't go wrong in the success method! try: _edControlSlotFAILURE.call(_edPlugin) except Exception: EDVerbose.DEBUG( "EDControlPlugin.executeSynchronous: ERROR in slotFAILURE!" ) EDVerbose.writeErrorTrace()
def __init__(self): EDLogging.__init__(self) Thread.__init__(self) self.__edSlotPreProcess = EDSlot() self.__edSlotProcess = EDSlot() self.__edSlotPostProcess = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__edSlotFinallyProcess = EDSlot() self.__bIsFailure = False self.__bIsTimeOut = False self.__fTimeOutInSeconds = None self.__fDefaultTimeOutInSeconds = 600.0 self.__bIsAbort = False # Reference to the object which calls execute or executeSynchronous self.__edObject = None self.__lExtraTime = [] # list of extra allowed time for execution (in second) self.__bLogTiming = False
class EDPlugin(EDAction): """ This is the EDNA plugin main class An EDNA plugin class: - is a configurable entity - has a base name (<date>-<random number>-<base name>) - handles input/output data (setter, getter, checker) - has warning and error messages - has a base and a working directory (both are configurable) The working directory is the folder from which the plugin is launched and should contain all associated files with the plugin execution (edna xml input/output, 3rd party output files) The base directory is the parent directory of the working directory. Example: the working directory of a control plugin is the base directory of the plugins that it invokes, i.e. the plugins working directories has the control plugin working directory as parent. - defines the method that generates an executive summary (user-related output summary) that sub-classes should implement """ CONF_BASE_DIR_LABEL = "baseDirectory" CONF_WORKING_DIR_LABEL = "workingDirectory" CONF_TIME_OUT = "timeOut" CONF_WRITE_XML_INPUT_OUTPUT = "writeXMLInputOutput" CONF_WRITE_XML_OUTPUT = "writeXMLOutput" CONF_WRITE_XML_INPUT = "writeXMLInput" def __init__ (self): """ Initializes plugin related attributes described above """ EDAction.__init__(self) self.__xsPluginItem = None self.__dictXSDataInputClass = {} self.__dictXSDataInput = {} self.__dictXSDataOutput = {} self.__strDefaultInputDataKey = "defaultInputData" self.__strDefaultOutputDataKey = "defaultOutputData" self.__edSlotExportDataOutput = EDSlot() self.__strBaseDirectory = None self.__strWorkingDirectory = None self.__strBaseName = None self.__listExecutiveSummaryLines = [] self.__strExecutiveSummarySeparator = "-" * 80 self.__listErrorMessages = [] self.__listWarningMessages = [] self.__isRequiredToHaveConfiguration = False self.__bWriteDataXMLInputOutput = True self.__bWriteDataXMLOutput = True self.__bWriteDataXMLInput = True self.__strPluginId = "%s-%08i" % (self.getClassName(), self.getId()) self.strPathDataInput = None self.strPathDataOutput = None def preProcess(self, _edObject=None): """ Writes xml data input in the working dir (if required) Connects a slot for generating the executive summary after the plugin execution Connects a slot for checking output data to the finally process Initialize the base directory Configures the plugin Checks the input data """ EDAction.preProcess(self, _edObject) self.DEBUG("EDPlugin.preProcess") self.connectPostProcess(self.exportDataOutput) if self.__bWriteDataXMLInputOutput: if self.__bWriteDataXMLInput: self.connectPreProcess(self.writeDataInput) self.connectPostProcess(self.generateExecutiveSummary) self.connectFinallyProcess(self.checkDataOutput) if (self.__strBaseName is None): self.setBaseName(self.createBaseName()) EDStatus.tellRunning(self.__strPluginId) self.connectFinallyProcess(self.tellFinished) self.checkParameters() def tellFinished(self, _edObject=None): """ Tell EDStatus that the plugin has finished, either in success either in error """ if self.isFailure(): EDStatus.tellFailure(self.__strPluginId) else: EDStatus.tellSuccess(self.__strPluginId) def checkDataOutput(self, _edObject=None): """ Checks if output data is available, if not issues a warning and sets an empty XSDataResult as output data Writes xml data output in the working dir (if required) """ EDAction.finallyProcess(self, _edObject) if self.__dictXSDataOutput == {}: strWarningMessage = "Output data for plugin %s not set, using XSDataResult as output" % self.getPluginName() self.WARNING(strWarningMessage) self.addWarningMessage(strWarningMessage) self.setDataOutput(XSDataResult()) if self.__bWriteDataXMLInputOutput: if self.__bWriteDataXMLOutput: self.writeDataOutput() def synchronize(self): """ This method calls EDAction.synchronize and if a time-out occurs an error message is added to the list of error messages. """ EDAction.synchronize(self) if self.isTimeOut(): strErrorMessage = "Timeout when waiting for %s to terminate." % self.getClassName() self.addErrorMessage(strErrorMessage) def setConfiguration(self, _xsPluginItem): """ Receives an XSPluginItem (Plugin Configuration) from the application """ self.DEBUG("EDPlugin.setConfiguration") self.__xsPluginItem = _xsPluginItem def getConfiguration(self): """ Gets the Plugin Configuration as an XSPluginItem """ self.DEBUG("EDPlugin.getConfiguration") return self.__xsPluginItem configuration = property(getConfiguration, setConfiguration) def getStringConfigurationParameterValue(self, _strConfigurationParameterName): """ This method returns a configuration parameter value if a corresponding configuration parameter name can be found in the configuration file. If an application wide configuration file is provided via EDApplication it will override the product configuration file. The configuration parameter is then searched in the configration file in following order: - If a plugin configuration item exists and the configration parameter name is present it will be used. - Otherwise if a product-wide (e.g. "mxPluginExec") configuration value exists it will be used. """ self.DEBUG("EDPlugin.getConfigurationParameterValue") strParameteValue = None xsPluginItem = self.getConfiguration() bFoundConfigurationParameter = False if xsPluginItem: strParameteValue = EDConfiguration.getParamItem(xsPluginItem, _strConfigurationParameterName).getValue() if strParameteValue: bFoundConfigurationParameter = True if not bFoundConfigurationParameter: xsPluginItem = EDApplication.getApplicationPluginConfiguration(self.getPluginName()) if xsPluginItem: strParameteValue = EDConfiguration.getParamItem(xsPluginItem, _strConfigurationParameterName).getValue() if strParameteValue: bFoundConfigurationParameter = True if not bFoundConfigurationParameter: xsPluginItem = EDApplication.getProjectPluginConfiguration(self.getPluginName()) if xsPluginItem: strParameteValue = EDConfiguration.getParamItem(xsPluginItem, _strConfigurationParameterName).getValue() if strParameteValue: bFoundConfigurationParameter = True return strParameteValue def getDoubleConfigurationParameterValue(self, _strConfigurationParameterName): fParameterValue = None strParameterValue = self.getStringConfigurationParameterValue(_strConfigurationParameterName) if strParameterValue: fParameterValue = float(strParameterValue) return fParameterValue def getIntegerConfigurationParameterValue(self, _strConfigurationParameterName): iParameterValue = None strParameterValue = self.getStringConfigurationParameterValue(_strConfigurationParameterName) if strParameterValue: iParameterValue = int(strParameterValue) return iParameterValue def configure(self): """ Should be overridden by the Final Plugin If needed This method should set its proper members attributes from a Plugin configuration Object """ self.DEBUG("EDPlugin.configure : %s" % self.getClassName()) xsPluginItem = self.getConfiguration() if xsPluginItem is None: xsPluginItem = EDApplication.getApplicationPluginConfiguration(self.getPluginName()) if (xsPluginItem is None): # No application wide configuration file found! Try to find a project specific config file: xsPluginItem = EDApplication.getProjectPluginConfiguration(self.getPluginName()) if (xsPluginItem is None): self.DEBUG("EDPlugin.configure: No plugin configuration found for " + self.getPluginName()) xsPluginItem = XSPluginItem() else: self.setConfiguration(xsPluginItem) # set Timeout if different from default one if self.getTimeOut() == self.getDefaultTimeOut(): # Try to get time out from plugin configuration iTimeOut = EDConfiguration.getIntegerParamValue(xsPluginItem, EDPlugin.CONF_TIME_OUT) if iTimeOut is not None: self.DEBUG("EDPlugin.configure: Setting time out to %d s from plugin configuration." % iTimeOut) self.setTimeOut(iTimeOut) else: self.DEBUG("EDPlugin.configure: Working directory already set before plugin is configured.") # Base directory strBaseDirectory = self.getBaseDirectory() if (strBaseDirectory is None): # Try to get base directory from plugin configuration strBaseDirectory = EDConfiguration.getStringParamValue(xsPluginItem, EDPlugin.CONF_BASE_DIR_LABEL) if(strBaseDirectory is None): # Try to get working directory from environment variable strBaseDirectory = os.environ.get("EDNA_BASE_DIRECTORY") if (strBaseDirectory is None): self.DEBUG("EDPlugin.configure: Using current base directory as working directory.") strBaseDirectory = os.getcwd() else: self.DEBUG("EDPlugin.configure: Setting base directory from $EDNA_WORKING_DIRECTORY.") else: if (strBaseDirectory == "."): self.DEBUG("EDPlugin.configure: Using current base directory as working directory.") strBaseDirectory = os.getcwd() else: strBaseDirectory = os.path.abspath(strBaseDirectory) self.DEBUG("EDPlugin.configure: Setting base directory from plugin configuration.") self.setBaseDirectory(strBaseDirectory) else: self.DEBUG("EDPlugin.configure: Base directory already set before plugin is configured.") # Working directory strWorkingDirectory = self.getWorkingDirectory() if (strWorkingDirectory is None): # Try to get working directory from plugin configuration strWorkingDirectory = EDConfiguration.getStringParamValue(xsPluginItem, EDPlugin.CONF_WORKING_DIR_LABEL) if(strWorkingDirectory is not None): self.DEBUG("EDPlugin.configure: Setting working directory from plugin configuration.") else: self.DEBUG("EDPlugin.configure: Setting working directory as base directory + base name.") strWorkingDirectory = os.path.join(self.getBaseDirectory(), self.getBaseName()) self.setWorkingDirectory(strWorkingDirectory) else: self.DEBUG("EDPlugin.configure: Working directory already set before plugin is configured.") # strWriteXMLInputOutput = EDConfiguration.getStringParamValue(xsPluginItem, EDPlugin.CONF_WRITE_XML_INPUT_OUTPUT) if strWriteXMLInputOutput != None: if strWriteXMLInputOutput.lower().strip() in ["false", "0"]: self.__bWriteDataXMLInputOutput = False else: self.__bWriteDataXMLInputOutput = True strWriteXMLOutput = EDConfiguration.getStringParamValue(xsPluginItem, EDPlugin.CONF_WRITE_XML_OUTPUT) if strWriteXMLOutput != None: if strWriteXMLOutput.lower().strip() in ["false", "0"]: self.__bWriteDataXMLOutput = False else: self.__bWriteDataXMLOutput = True strWriteXMLInput = EDConfiguration.getStringParamValue(xsPluginItem, EDPlugin.CONF_WRITE_XML_INPUT) if strWriteXMLInput != None: if strWriteXMLInput.lower().strip() in ["false", "0"]: self.__bWriteDataXMLInput = False else: self.__bWriteDataXMLInput = True def execute(self, _edObject=None): # Configure the plugin before starting it's thread self.configure() EDAction.execute(self, _edObject) def checkParameters(self): """ Should be overridden by the Final Plugin If needed This method should check that the data input are consistent """ self.DEBUG("EDPlugin.checkParameters") def setXSDataInputClass(self, _xsDataInputClass, _strDataInputKey=None): """ This method should be called in the constructor of the derived plugins in order to set the XSData type of the input data, e.g. XSDataInputXXX """ strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInputClass.keys()): strErrorMessage = "ERROR: " + self.getPluginName() + ".setXSDataInputClass, Data Input Class already defined for key: " + strDataInputKey self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError, strErrorMessage self.__dictXSDataInputClass[ strDataInputKey ] = _xsDataInputClass def getXSDataInputClass(self, _strDataInputKey=None): """ Returns the XSData type of the input data. """ pyXSDataInputClass = None strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInputClass.keys()): pyXSDataInputClass = self.__dictXSDataInputClass[ strDataInputKey ] return pyXSDataInputClass def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the plugin input data. _oDataInput could be either an String XML or an XSData object. The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ self.DEBUG("EDPlugin.setDataInput") strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey # Allow for None input if (_oDataInput is None): self.DEBUG("EDPlugin.setDataInput: Data input is None") self.__dictXSDataInput[ strDataInputKey ] = [] elif (self.getXSDataInputClass(strDataInputKey) is None): strErrorMessage = "ERROR: " + self.getPluginName() + ".setDataInput, Data Input Class not defined for key: " + strDataInputKey self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError, strErrorMessage else: # Check the type xsDataInput = None if isinstance(_oDataInput, (str, unicode)): self.DEBUG("EDPlugin.setDataInput: Input Data is string ") xsDataInput = self.getXSDataInputClass(strDataInputKey).parseString(_oDataInput) elif (isinstance(_oDataInput, self.getXSDataInputClass(strDataInputKey))): self.DEBUG("EDPlugin.setDataInput: Input Data is of type " + str(_oDataInput.__class__)) xsDataInput = _oDataInput else: strErrorMessage = "ERROR: %s.setDataInput, wrong data type %r for data input key: %s, expected XML string or %r" % \ (self.getPluginName(), _oDataInput.__class__, strDataInputKey, self.getXSDataInputClass(strDataInputKey)) self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError, strErrorMessage # Add the object to a list if its key is not the default key if (strDataInputKey != self.__strDefaultInputDataKey) : # Check if there's already a list stored if (not strDataInputKey in self.__dictXSDataInput.keys()): self.__dictXSDataInput[ strDataInputKey ] = [] self.__dictXSDataInput[ strDataInputKey ].append(xsDataInput) else: self.__dictXSDataInput[ strDataInputKey ] = xsDataInput def hasDataInput(self, _strDataInputKey=None): """ Returns True if the plugin has Input Data for a particular key. If the key is not provided a default key is used. """ strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInput.keys()): return True else: return False def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ oValue = None strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInput.keys()): oValue = self.__dictXSDataInput[ strDataInputKey ] else: strErrorMessage = self.getPluginName() + ".getDataInput, no input data defined for key: " + strDataInputKey self.warning(strErrorMessage) self.addWarningMessage(strErrorMessage) return oValue def delDataInput(self, _strDataInputKey=None): """ Deletes the data input for a particular key. If the key is not provided a default key is used. """ strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInput.keys()): self.__dictXSDataInput[ strDataInputKey ] = None else: strErrorMessage = self.getPluginName() + ".delDataInput, no input data defined for key: " + strDataInputKey self.warning(strErrorMessage) self.addWarningMessage(strErrorMessage) # Property for dataInput dataInput = property(getDataInput, setDataInput, delDataInput, "Property for dataInput") def setDataOutput(self, _xsDataOutput, _strDataOutputKey=None): """ Sets the plugin output data for a particular key. If the key is not provided a default key is used. If the key is already defined in the dictionary, the corresponding data object is added to a list which contains the already stored object(s). """ strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey # Add the object to a list if its key not the default key if (strDataOutputKey == self.__strDefaultOutputDataKey): self.__dictXSDataOutput[ strDataOutputKey ] = _xsDataOutput else: # Check if the _xsDataoutput object is already a list if (type(_xsDataOutput) == type ([])): self.__dictXSDataOutput[ strDataOutputKey ] = _xsDataOutput else: # Check if the stored object contains already a list if (not strDataOutputKey in self.__dictXSDataOutput.keys()): self.__dictXSDataOutput[ strDataOutputKey ] = [] self.__dictXSDataOutput[ strDataOutputKey ].append(_xsDataOutput) def getDataOutput(self, _strDataOutputKey=None): """ Returns the Plugin Output Data """ oValue = None strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey if (strDataOutputKey in self.__dictXSDataOutput.keys()): oValue = self.__dictXSDataOutput[ strDataOutputKey ] return oValue def hasDataOutput(self, _strDataOutputKey=None): """ Returns True if the plugin has the specified Output Data """ strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey if (strDataOutputKey in self.__dictXSDataOutput.keys()): return True else: return False def delDataOutput(self, _strDataOutputKey=None): """ Deletes the data output for a particular key. If the key is not provided a default key is used. """ strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey if (strDataOutputKey in self.__dictXSDataOutput.keys()): self.__dictXSDataOutput[ strDataOutputKey ] = None else: strErrorMessage = self.getPluginName() + ".delDataOutput, no output data defined for key: " + strDataInputKey self.warning(strErrorMessage) self.addWarningMessage(strErrorMessage) # Property for dataOutput dataOutput = property(getDataOutput, setDataOutput, delDataOutput, "Property for dataOutput") def exportDataOutput(self, _edPlugin=None): """ Deprecated Exports the Plugin Output Data to slot """ self.DEBUG("EDPlugin.exportDataOutput") self.__edSlotExportDataOutput.call(self.__dictXSDataOutput) def connectExportDataOutput(self, _oMethod): """ Deprecated """ self.synchronizeOn() if (_oMethod != None): self.__edSlotExportDataOutput.connect(_oMethod) self.synchronizeOff() def generateExecutiveSummary(self, _edPlugin): """ This method, which should be implemented by sub-classes, generates an executive summary (user-related output summary). """ self.DEBUG("EDPlugin.generateExecutiveSummary") def addErrorMessage(self, _strErrorMessage): """ Adds an error message to the error messages list """ self.__listErrorMessages.append(_strErrorMessage) def getErrorMessages(self): """ Returns the error messages list OBS! This method is deprecated, please use getListOfErrorMessages instead. """ self.warning("Deprecation by Monday 7th June 2010 of EDPlugin, called getErrorMessages") from EDImportLib import EDList return EDList(self.__listErrorMessages) def getListOfErrorMessages(self): """ Returns the error messages list """ return self.__listErrorMessages def addWarningMessage(self, _strWarningMessage): """ Adds a warning message to the warning messages list """ self.DEBUG("EDPlugin.addWarningMessage : " + _strWarningMessage) self.__listWarningMessages.append(_strWarningMessage) def getWarningMessages(self): """ Returns the warning messages list OBS! This method is deprecated, please use getListOfWarningMessages instead. """ self.warning("Deprecation by Monday 7th June 2010 of EDPlugin, called getWarningMessages") from EDImportLib import EDList return EDList(self.__listWarningMessages) def getListOfWarningMessages(self): """ Returns the warning messages list """ return self.__listWarningMessages def writeDataInput(self, _edObject=None): """ Writes the input data object(s) into a working dir xml file """ self.DEBUG("EDPlugin.writeDataInput") strBasename = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName())) for strKey in self.__dictXSDataInput.keys(): if (strKey == self.__strDefaultInputDataKey): # "Old" style xsDataInput = self.__dictXSDataInput[ self.__strDefaultInputDataKey ] self.strPathDataInput = strBasename + "_dataInput.xml" EDUtilsFile.writeFile(self.strPathDataInput, xsDataInput.marshal()) else: # We have a list of objects listXSDataInput = self.__dictXSDataInput[ strKey ] for iIndex, xsDataInput in enumerate(listXSDataInput): strPathDataInput = "%s_%s_%d_dataInput.xml" % (strBasename, strKey, iIndex) EDUtilsFile.writeFile(strPathDataInput, xsDataInput.marshal()) def writeDataOutput(self, _edObject=None): """ Writes the output data object(s) into a working dir xml file """ self.DEBUG("EDPlugin.writeDataOutput") for strKey in self.__dictXSDataOutput.keys(): if (strKey == self.__strDefaultOutputDataKey): # "Old" style xsDataOutput = self.__dictXSDataOutput[ self.__strDefaultOutputDataKey ] if (xsDataOutput is not None): self.strPathDataOutput = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()) + "_dataOutput.xml") EDUtilsFile.writeFile(self.strPathDataOutput, xsDataOutput.marshal()) else: listXSDataOutput = self.__dictXSDataOutput[ strKey ] for iIndex, xsDataOutput in enumerate(listXSDataOutput): if (xsDataOutput is not None): strPathDataOutput = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()) + "_" + strKey + "_%d_dataOutput.xml" % iIndex) EDUtilsFile.writeFile(strPathDataOutput, xsDataOutput.marshal()) def getBaseName(self): """ Returns the plugin base name """ if (self.__strBaseName is None): self.__strBaseName = self.createBaseName() return self.__strBaseName def setBaseName(self, _strBaseName): """ Sets the plugin base name """ self.__strBaseName = self.compactPluginName(_strBaseName) self.setName(self.__strBaseName) # Create the directory baseDirectory/baseName which will be used as working directory strWorkingDirPath = os.path.join(self.getBaseDirectory(), self.__strBaseName) if not os.path.isdir(strWorkingDirPath): os.mkdir(strWorkingDirPath) self.setWorkingDirectory(strWorkingDirPath) def createBaseName(self): """ Generates the plugin base name: (<prefix>-<object ID>) """ # First try to use global instance ID from EDObject strBaseName = "%s-%08d" % (self.compactPluginName(self.getPluginName()), self.getId()) strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName) # Try to create the directory... try: os.mkdir(strBaseDir) except BaseException, strErrorDetail: self.error("EDPlugin.createBaseName: Could not create base directory %s because of %s" % (strBaseDir, strErrorDetail)) self.warning("EDPlugin.createBaseName: Trying to create alternative base directory...") self.writeErrorTrace() strTempDir = tempfile.mkdtemp(prefix=strBaseName, dir=self.getBaseDirectory()) os.chmod(strTempDir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) strBaseName = os.path.split(strTempDir)[1] strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName) self.warning("EDPlugin.createBaseName: Alternative base directory created: %s" % strBaseDir) self.DEBUG("EDPlugin.createBaseName : Directory created = " + strBaseDir) return strBaseName
class EDJob(EDObject): PLUGIN_STATE_UNITIALIZED = "uninitialized" PLUGIN_STATE_RUNNING = "running" PLUGIN_STATE_SUCCESS = "success" PLUGIN_STATE_FAILURE = "failure" __edFactoryPlugin = EDFactoryPlugin() __dictJobs = {} __semaphore = threading.Semaphore() __dictPluginLastId = {} __fStartTime = time.time() def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDObject.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__bXmlInputSet = False self.__status = None # self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName) self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName) EDJob.__semaphore.acquire() # Create the JobID if not self.__strPluginName in EDJob.__dictPluginLastId: EDJob.__dictPluginLastId[_strPluginName] = 0 else: EDJob.__dictPluginLastId[_strPluginName] += 1 self.__jobId = "%s-%i" % (self.__strPluginName, EDJob.__dictPluginLastId[_strPluginName]) EDJob.__dictJobs[self.__jobId] = self EDJob.__semaphore.release() if (self.__edPlugin is None): EDVerbose.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the job (plugin) input data. @param: _oDataInput: could be either an String XML or an XSData object. @param _strDataInputKey: the key of an input data dictionnary The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ if _oDataInput in ["", None]: self.__bXmlInputSet = False return self.synchronizeOn() if (self.__edPlugin is not None): self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey) self.__bXmlInputSet = True else: EDVerbose.WARNING("Setting DataInput for uninstanciated plugin %s." % self.__strPluginName) self.synchronizeOff() def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ if (self.__edPlugin is not None): return self.__edPlugin.getDataInput(_strDataInputKey) else: EDVerbose.WARNING("Getting DataInput for uninstanciated plugin %s." % self.__strPluginName) def getDataOutput(self, _strDataOutputKey=None): """ Returns the Plugin Output Data """ if (self.__edPlugin is not None): return self.__edPlugin.getDataOutput(_strDataOutputKey) else: EDVerbose.WARNING("Getting DataOutput for uninstanciated plugin %s." % self.__strPluginName) def execute(self): """ Launch the EDNA plugin @return: JobId @rtype: string """ returnId = None if not self.__bXmlInputSet: EDVerbose.WARNING("Not executing job %s as input is empty" % self.__jobId) if (self.__edPlugin is not None): self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_RUNNING self.__edPlugin.connectSUCCESS(self.successPluginExecution) self.__edPlugin.connectFAILURE(self.failurePluginExecution) self.__edPlugin.execute() returnId = self.__jobId self.synchronizeOff() else: EDVerbose.WARNING("Trying to run a plugin that does not exist: %s " % self.__strPluginName) return returnId def synchronize(self): """ Synchronize the execution of the job with the calling thread. """ self.__edPlugin.synchronize() def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_SUCCESS EDVerbose.screen("Plugin %s execution ended with success" % self.__jobId) self.synchronizeOff() try: self.__edSlotSUCCESS.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Success call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Common call-back (after success) for %s" % self.__jobId) EDVerbose.writeErrorTrace() def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_FAILURE EDVerbose.screen("Plugin %s execution ended with failure" % self.__jobId) self.synchronizeOff() try: self.__edSlotFAILURE.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Failure call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Common call-back (after failure) for %s" % self.__jobId) EDVerbose.writeErrorTrace() def connectSUCCESS(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotSUCCESS.connect(_oMethod) self.synchronizeOff() def connectFAILURE(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotFAILURE.connect(_oMethod) self.synchronizeOff() def connectCallBack(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotCallBack.connect(_oMethod) self.synchronizeOff() def getJobId(self): """ @return: JobId i.e. EDPluginName-Number @rtype: string """ return self.__jobId def getPluginName(self): """ @return: Name of the plugin @rtype: string """ return self.__strPluginName def getPlugin(self): """ @return: the plugin (instance) @rtype: python object """ return self.__edPlugin def getStatus(self): """ @return: status of the Job @rtype: string """ return self.__status def getMemSize(self): """ try to guess the size in memory of a job @return: expected size in memory """ if asizeof is not None: return asizeof.asizeof(self) @staticmethod def getStatusFromID(jobId): """ Retrieve the job (hence the plugin) status @param jobId: the Job identification number @type jobId: string @return: the EDJob status @rtype: string """ if jobId in EDJob.__dictJobs: return EDJob.__dictJobs[jobId].getStatus() else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) @staticmethod def getJobFromID(jobId): """ Retrieve the job (hence the plugin) @param jobId: the Job identification number @type jobId: string @return: the "EDJob instance", which contains the plugin (__edPlugin) and the status @rtype: a Python object. """ if jobId in EDJob.__dictJobs: return EDJob.__dictJobs[jobId] else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) @staticmethod def getMemoryFootprint(): if asizeof is not None: return asizeof.asizesof(EDJob.__dictJobs) @staticmethod def stats(): """ retrieve some statistics """ output = [] fExecTime = time.time() - EDJob.__fStartTime keys = EDJob.__dictJobs.keys() for key in keys : num = int(key.split("-", 1)[1]) job = EDJob.__dictJobs[key] output.append([num, job.getPluginName(), job.getStatus(), job.getPlugin().getRunTime(), job.getMemSize()]) output.sort() iNbJob = max(1, len(output)) EDVerbose.screen("%s\t|\t%s\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("id", "EDPluginName", "status", "runtime", "memory")) fWall = 0.0 fSumProd = 0.0 fSumX = 0.0 fSumX2 = 0.0 for oneJob in output: fWall += oneJob[3] fSumX += oneJob[0] fSumX2 += oneJob[0] * oneJob[0] fSumProd += oneJob[0] * oneJob[3] EDVerbose.screen("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob)) EDVerbose.screen("_" * 90) EDVerbose.screen("Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime)) EDVerbose.screen("Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob)) fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX) fOrd = (fWall - fSlope * fSumX) / iNbJob EDVerbose.screen("Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope))
class EDJob(EDObject): PLUGIN_STATE_UNITIALIZED = "uninitialized" PLUGIN_STATE_RUNNING = "running" PLUGIN_STATE_SUCCESS = "success" PLUGIN_STATE_FAILURE = "failure" __edFactoryPlugin = EDFactoryPlugin() __dictJobs = {} __semaphore = threading.Semaphore() __dictPluginLastId = {} __fStartTime = time.time() def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDObject.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__bXmlInputSet = False self.__status = None # self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName) self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName) EDJob.__semaphore.acquire() # Create the JobID if not self.__strPluginName in EDJob.__dictPluginLastId: EDJob.__dictPluginLastId[_strPluginName] = 0 else: EDJob.__dictPluginLastId[_strPluginName] += 1 self.__jobId = "%s-%i" % (self.__strPluginName, EDJob.__dictPluginLastId[_strPluginName]) EDJob.__dictJobs[self.__jobId] = self EDJob.__semaphore.release() if (self.__edPlugin is None): EDVerbose.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the job (plugin) input data. @param: _oDataInput: could be either an String XML or an XSData object. @param _strDataInputKey: the key of an input data dictionnary The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ if _oDataInput in ["", None]: self.__bXmlInputSet = False return self.synchronizeOn() if (self.__edPlugin is not None): self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey) self.__bXmlInputSet = True else: EDVerbose.WARNING( "Setting DataInput for uninstanciated plugin %s." % self.__strPluginName) self.synchronizeOff() def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ if (self.__edPlugin is not None): return self.__edPlugin.getDataInput(_strDataInputKey) else: EDVerbose.WARNING( "Getting DataInput for uninstanciated plugin %s." % self.__strPluginName) def getDataOutput(self, _strDataOutputKey=None): """ Returns the Plugin Output Data """ if (self.__edPlugin is not None): return self.__edPlugin.getDataOutput(_strDataOutputKey) else: EDVerbose.WARNING( "Getting DataOutput for uninstanciated plugin %s." % self.__strPluginName) def execute(self): """ Launch the EDNA plugin @return: JobId @rtype: string """ returnId = None if not self.__bXmlInputSet: EDVerbose.WARNING("Not executing job %s as input is empty" % self.__jobId) if (self.__edPlugin is not None): self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_RUNNING self.__edPlugin.connectSUCCESS(self.successPluginExecution) self.__edPlugin.connectFAILURE(self.failurePluginExecution) self.__edPlugin.execute() returnId = self.__jobId self.synchronizeOff() else: EDVerbose.WARNING( "Trying to run a plugin that does not exist: %s " % self.__strPluginName) return returnId def synchronize(self): """ Synchronize the execution of the job with the calling thread. """ self.__edPlugin.synchronize() def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_SUCCESS EDVerbose.screen("Plugin %s execution ended with success" % self.__jobId) self.synchronizeOff() try: self.__edSlotSUCCESS.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Success call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR( "Error in execution of Common call-back (after success) for %s" % self.__jobId) EDVerbose.writeErrorTrace() def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_FAILURE EDVerbose.screen("Plugin %s execution ended with failure" % self.__jobId) self.synchronizeOff() try: self.__edSlotFAILURE.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Failure call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR( "Error in execution of Common call-back (after failure) for %s" % self.__jobId) EDVerbose.writeErrorTrace() def connectSUCCESS(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotSUCCESS.connect(_oMethod) self.synchronizeOff() def connectFAILURE(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotFAILURE.connect(_oMethod) self.synchronizeOff() def connectCallBack(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotCallBack.connect(_oMethod) self.synchronizeOff() def getJobId(self): """ @return: JobId i.e. EDPluginName-Number @rtype: string """ return self.__jobId def getPluginName(self): """ @return: Name of the plugin @rtype: string """ return self.__strPluginName def getPlugin(self): """ @return: the plugin (instance) @rtype: python object """ return self.__edPlugin def getStatus(self): """ @return: status of the Job @rtype: string """ return self.__status def getMemSize(self): """ try to guess the size in memory of a job @return: expected size in memory """ if asizeof is not None: return asizeof.asizeof(self) @staticmethod def getStatusFromID(jobId): """ Retrieve the job (hence the plugin) status @param jobId: the Job identification number @type jobId: string @return: the EDJob status @rtype: string """ if jobId in EDJob.__dictJobs: return EDJob.__dictJobs[jobId].getStatus() else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) @staticmethod def getJobFromID(jobId): """ Retrieve the job (hence the plugin) @param jobId: the Job identification number @type jobId: string @return: the "EDJob instance", which contains the plugin (__edPlugin) and the status @rtype: a Python object. """ if jobId in EDJob.__dictJobs: return EDJob.__dictJobs[jobId] else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) @staticmethod def getMemoryFootprint(): if asizeof is not None: return asizeof.asizesof(EDJob.__dictJobs) @staticmethod def stats(): """ retrieve some statistics """ output = [] fExecTime = time.time() - EDJob.__fStartTime keys = EDJob.__dictJobs.keys() for key in keys: num = int(key.split("-", 1)[1]) job = EDJob.__dictJobs[key] output.append([ num, job.getPluginName(), job.getStatus(), job.getPlugin().getRunTime(), job.getMemSize() ]) output.sort() iNbJob = max(1, len(output)) EDVerbose.screen("%s\t|\t%s\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("id", "EDPluginName", "status", "runtime", "memory")) fWall = 0.0 fSumProd = 0.0 fSumX = 0.0 fSumX2 = 0.0 for oneJob in output: fWall += oneJob[3] fSumX += oneJob[0] fSumX2 += oneJob[0] * oneJob[0] fSumProd += oneJob[0] * oneJob[3] EDVerbose.screen("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob)) EDVerbose.screen("_" * 90) EDVerbose.screen( "Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime)) EDVerbose.screen( "Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob)) fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX) fOrd = (fWall - fSlope * fSumX) / iNbJob EDVerbose.screen( "Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope))
class EDPlugin(EDAction): """ This is the EDNA plugin main class An EDNA plugin class: - is a configurable entity - has a base name (<date>-<random number>-<base name>) - handles input/output data (setter, getter, checker) - has warning and error messages - has a base and a working directory (both are configurable) The working directory is the folder from which the plugin is launched and should contain all associated files with the plugin execution (edna xml input/output, 3rd party output files) The base directory is the parent directory of the working directory. Example: the working directory of a control plugin is the base directory of the plugins that it invokes, i.e. the plugins working directories has the control plugin working directory as parent. - defines the method that generates an executive summary (user-related output summary) that sub-classes should implement """ CONF_BASE_DIR_LABEL = "baseDirectory" CONF_WORKING_DIR_LABEL = "workingDirectory" CONF_TIME_OUT = "timeOut" CONF_WRITE_XML_INPUT_OUTPUT = "writeXMLInputOutput" CONF_WRITE_XML_OUTPUT = "writeXMLOutput" CONF_WRITE_XML_INPUT = "writeXMLInput" def __init__ (self): """ Initializes plugin related attributes described above """ EDAction.__init__(self) self.__xsPluginItem = None self.__dictXSDataInputClass = {} self.__dictXSDataInput = {} self.__dictXSDataOutput = {} self.__strDefaultInputDataKey = "defaultInputData" self.__strDefaultOutputDataKey = "defaultOutputData" self.__edSlotExportDataOutput = EDSlot() self.__strBaseDirectory = None self.__strWorkingDirectory = None self.__strBaseName = None self.__listExecutiveSummaryLines = [] self.__strExecutiveSummarySeparator = "-" * 80 self.__listErrorMessages = [] self.__listWarningMessages = [] self.__isRequiredToHaveConfiguration = False self.__bWriteDataXMLInputOutput = True self.__bWriteDataXMLOutput = True self.__bWriteDataXMLInput = True self.__strPluginId = "%s-%08i" % (self.getClassName(), self.getId()) self.strPathDataInput = None self.strPathDataOutput = None self.__bUseWarningInsteadOfError = False self.__edConfiguration = EDConfigurationStatic() def preProcess(self, _edObject=None): """ Writes xml data input in the working dir (if required) Connects a slot for generating the executive summary after the plugin execution Connects a slot for checking output data to the finally process Initialize the base directory Configures the plugin Checks the input data """ EDAction.preProcess(self, _edObject) self.DEBUG("EDPlugin.preProcess") self.connectPostProcess(self.exportDataOutput) if self.__bWriteDataXMLInputOutput: if self.__bWriteDataXMLInput: self.connectPreProcess(self.writeDataInput) self.connectPostProcess(self.generateExecutiveSummary) self.connectFinallyProcess(self.checkDataOutput) if (self.__strBaseName is None): self.setBaseName(self.createBaseName()) EDStatus.tellRunning(self.__strPluginId) self.connectFinallyProcess(self.tellFinished) self.checkParameters() def tellFinished(self, _edObject=None): """ Tell EDStatus that the plugin has finished, either in success either in error """ if self.isFailure(): EDStatus.tellFailure(self.__strPluginId) else: EDStatus.tellSuccess(self.__strPluginId) def checkDataOutput(self, _edObject=None): """ Checks if output data is available, if not issues a warning and sets an empty XSDataResult as output data Writes xml data output in the working dir (if required) """ EDAction.finallyProcess(self, _edObject) if self.__dictXSDataOutput == {}: strWarningMessage = "Output data for plugin %s not set, using XSDataResult as output" % self.getPluginName() self.WARNING(strWarningMessage) self.addWarningMessage(strWarningMessage) self.setDataOutput(XSDataResult()) if self.__bWriteDataXMLInputOutput: if self.__bWriteDataXMLOutput: self.writeDataOutput() def synchronize(self): """ This method calls EDAction.synchronize and if a time-out occurs an error message is added to the list of error messages. """ EDAction.synchronize(self) if self.isTimeOut(): strErrorMessage = "Timeout when waiting for %s to terminate." % self.getClassName() self.addErrorMessage(strErrorMessage) def getConfig(self): """ Gets the Plugin Configuration as a dictionary """ # self.DEBUG("EDPlugin.getConfig") return self.__edConfiguration.get(self.getPluginName(), {}) def setConfig(self, _dict, _bLocal=False): """ Receives a dictionary (Plugin Configuration) from the application. """ self.DEBUG("EDPlugin.setConfiguration") if _bLocal: self.__edConfiguration = EDConfiguration() if _dict is not None: self.__edConfiguration[self.getPluginName()] = _dict else: self.__edConfiguration[self.getPluginName()] = {} config = property(getConfig, setConfig) def configure(self): """ Should be overridden by the Final Plugin If needed This method should set its proper members attributes from a Plugin configuration Object """ self.DEBUG("EDPlugin.configure : plugin name = %s, EDNA_SITE = %s" % (self.getPluginName(), EDUtilsPath.EDNA_SITE)) # set Timeout if different from default one if self.getTimeOut() == self.getDefaultTimeOut(): # Try to get time out from plugin configuration iTimeOut = self.config.get(EDPlugin.CONF_TIME_OUT, None) if iTimeOut is not None: self.DEBUG("EDPlugin.configure: Setting time out to %d s from plugin configuration." % iTimeOut) self.setTimeOut(iTimeOut) else: self.DEBUG("EDPlugin.configure: timeout already set before plugin is configured.") # Base directory strBaseDirectory = self.getBaseDirectory() if (strBaseDirectory is None): # Try to get base directory from plugin configuration strBaseDirectory = self.config.get(EDPlugin.CONF_BASE_DIR_LABEL, None) if(strBaseDirectory is None): # Try to get working directory from environment variable strBaseDirectory = os.environ.get("EDNA_BASE_DIRECTORY") if (strBaseDirectory is None): self.DEBUG("EDPlugin.configure: Using current base directory as working directory.") strBaseDirectory = os.getcwd() else: self.DEBUG("EDPlugin.configure: Setting base directory from $EDNA_WORKING_DIRECTORY.") else: if (strBaseDirectory == "."): self.DEBUG("EDPlugin.configure: Using current base directory as working directory.") strBaseDirectory = os.getcwd() else: strBaseDirectory = os.path.abspath(strBaseDirectory) self.DEBUG("EDPlugin.configure: Setting base directory from plugin configuration.") self.setBaseDirectory(strBaseDirectory) else: self.DEBUG("EDPlugin.configure: Base directory already set before plugin is configured.") # Working directory strWorkingDirectory = self.getWorkingDirectory() if (strWorkingDirectory is None): # Try to get working directory from plugin configuration strWorkingDirectory = self.config.get(EDPlugin.CONF_WORKING_DIR_LABEL, None) if(strWorkingDirectory is not None): self.DEBUG("EDPlugin.configure: Setting working directory from plugin configuration.") else: self.DEBUG("EDPlugin.configure: Setting working directory as base directory + base name.") strWorkingDirectory = os.path.join(self.getBaseDirectory(), self.getBaseName()) self.setWorkingDirectory(strWorkingDirectory) else: self.DEBUG("EDPlugin.configure: Working directory already set before plugin is configured.") # self.__bWriteDataXMLInputOutput = bool(self.config.get(self.CONF_WRITE_XML_INPUT_OUTPUT, True)) self.__bWriteDataXMLOutput = bool(self.config.get(self.CONF_WRITE_XML_OUTPUT, True)) self.__bWriteDataXMLInput = bool(self.config.get(self.CONF_WRITE_XML_INPUT, True)) def execute(self, _edObject=None): # Configure the plugin before starting it's thread self.configure() EDAction.execute(self, _edObject) def checkParameters(self): """ Should be overridden by the Final Plugin If needed This method should check that the data input are consistent """ self.DEBUG("EDPlugin.checkParameters") def setXSDataInputClass(self, _xsDataInputClass, _strDataInputKey=None): """ This method should be called in the constructor of the derived plugins in order to set the XSData type of the input data, e.g. XSDataInputXXX """ strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInputClass.keys()): strErrorMessage = "ERROR: " + self.getPluginName() + ".setXSDataInputClass, Data Input Class already defined for key: " + strDataInputKey self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError(strErrorMessage) self.__dictXSDataInputClass[ strDataInputKey ] = _xsDataInputClass def getXSDataInputClass(self, _strDataInputKey=None): """ Returns the XSData type of the input data. """ pyXSDataInputClass = None strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInputClass.keys()): pyXSDataInputClass = self.__dictXSDataInputClass[ strDataInputKey ] return pyXSDataInputClass def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the plugin input data. _oDataInput could be either an String XML or an XSData object. The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ self.DEBUG("EDPlugin.setDataInput") strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey # Allow for None input if (_oDataInput is None): self.DEBUG("EDPlugin.setDataInput: Data input is None") self.__dictXSDataInput[ strDataInputKey ] = [] elif (self.getXSDataInputClass(strDataInputKey) is None): strErrorMessage = "ERROR: " + self.getPluginName() + ".setDataInput, Data Input Class not defined for key: " + strDataInputKey self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError(strErrorMessage) else: # Check the type xsDataInput = None if isinstance(_oDataInput, (str, unicode)): self.DEBUG("EDPlugin.setDataInput: Input Data is string ") xsDataInput = self.getXSDataInputClass(strDataInputKey).parseString(_oDataInput) elif (isinstance(_oDataInput, self.getXSDataInputClass(strDataInputKey))): self.DEBUG("EDPlugin.setDataInput: Input Data is of type " + str(_oDataInput.__class__)) xsDataInput = _oDataInput else: strErrorMessage = "ERROR: %s.setDataInput, wrong data type %r for data input key: %s, expected XML string or %r" % \ (self.getPluginName(), _oDataInput.__class__, strDataInputKey, self.getXSDataInputClass(strDataInputKey)) self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError(strErrorMessage) # Add the object to a list if its key is not the default key if (strDataInputKey != self.__strDefaultInputDataKey) : # Check if there's already a list stored if (not strDataInputKey in self.__dictXSDataInput.keys()): self.__dictXSDataInput[ strDataInputKey ] = [] self.__dictXSDataInput[ strDataInputKey ].append(xsDataInput) else: self.__dictXSDataInput[ strDataInputKey ] = xsDataInput def hasDataInput(self, _strDataInputKey=None): """ Returns True if the plugin has Input Data for a particular key. If the key is not provided a default key is used. """ strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInput.keys()): return True else: return False def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ oValue = None strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInput.keys()): oValue = self.__dictXSDataInput[ strDataInputKey ] else: strErrorMessage = self.getPluginName() + ".getDataInput, no input data defined for key: " + strDataInputKey self.warning(strErrorMessage) self.addWarningMessage(strErrorMessage) return oValue def delDataInput(self, _strDataInputKey=None): """ Deletes the data input for a particular key. If the key is not provided a default key is used. """ strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInput.keys()): self.__dictXSDataInput[ strDataInputKey ] = None else: strErrorMessage = self.getPluginName() + ".delDataInput, no input data defined for key: " + strDataInputKey self.warning(strErrorMessage) self.addWarningMessage(strErrorMessage) # Property for dataInput dataInput = property(getDataInput, setDataInput, delDataInput, "Property for dataInput") def setDataOutput(self, _xsDataOutput, _strDataOutputKey=None): """ Sets the plugin output data for a particular key. If the key is not provided a default key is used. If the key is already defined in the dictionary, the corresponding data object is added to a list which contains the already stored object(s). """ strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey # Add the object to a list if its key not the default key if (strDataOutputKey == self.__strDefaultOutputDataKey): self.__dictXSDataOutput[ strDataOutputKey ] = _xsDataOutput else: # Check if the _xsDataoutput object is already a list if (type(_xsDataOutput) == list): self.__dictXSDataOutput[ strDataOutputKey ] = _xsDataOutput else: # Check if the stored object contains already a list if (not strDataOutputKey in self.__dictXSDataOutput.keys()): self.__dictXSDataOutput[ strDataOutputKey ] = [] self.__dictXSDataOutput[ strDataOutputKey ].append(_xsDataOutput) def getDataOutput(self, _strDataOutputKey=None): """ Returns the Plugin Output Data """ oValue = None strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey if (strDataOutputKey in self.__dictXSDataOutput.keys()): oValue = self.__dictXSDataOutput[ strDataOutputKey ] return oValue def hasDataOutput(self, _strDataOutputKey=None): """ Returns True if the plugin has the specified Output Data """ strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey if (strDataOutputKey in self.__dictXSDataOutput.keys()): return True else: return False def delDataOutput(self, _strDataOutputKey=None): """ Deletes the data output for a particular key. If the key is not provided a default key is used. """ strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey if (strDataOutputKey in self.__dictXSDataOutput.keys()): self.__dictXSDataOutput[ strDataOutputKey ] = None else: strErrorMessage = self.getPluginName() + ".delDataOutput, no output data defined for key: " + _strDataOutputKey self.warning(strErrorMessage) self.addWarningMessage(strErrorMessage) # Property for dataOutput dataOutput = property(getDataOutput, setDataOutput, delDataOutput, "Property for dataOutput") def exportDataOutput(self, _edPlugin=None): """ Deprecated Exports the Plugin Output Data to slot """ self.DEBUG("EDPlugin.exportDataOutput") self.__edSlotExportDataOutput.call(self.__dictXSDataOutput) def connectExportDataOutput(self, _oMethod): """ Deprecated """ self.synchronizeOn() if (_oMethod != None): self.__edSlotExportDataOutput.connect(_oMethod) self.synchronizeOff() def generateExecutiveSummary(self, _edPlugin): """ This method, which should be implemented by sub-classes, generates an executive summary (user-related output summary). """ self.DEBUG("EDPlugin.generateExecutiveSummary") def addErrorMessage(self, _strErrorMessage): """ Adds an error message to the error messages list """ self.__listErrorMessages.append(_strErrorMessage) def getListOfErrorMessages(self): """ Returns the error messages list """ return self.__listErrorMessages def addWarningMessage(self, _strWarningMessage): """ Adds a warning message to the warning messages list """ self.DEBUG("EDPlugin.addWarningMessage : " + _strWarningMessage) self.__listWarningMessages.append(_strWarningMessage) def getListOfWarningMessages(self): """ Returns the warning messages list """ return self.__listWarningMessages def writeDataInput(self, _edObject=None): """ Writes the input data object(s) into a working dir xml file """ self.DEBUG("EDPlugin.writeDataInput") strBasename = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName())) for strKey in self.__dictXSDataInput.keys(): if (strKey == self.__strDefaultInputDataKey): # "Old" style xsDataInput = self.__dictXSDataInput[ self.__strDefaultInputDataKey ] self.strPathDataInput = strBasename + "_dataInput.xml" EDUtilsFile.writeFile(self.strPathDataInput, xsDataInput.marshal()) else: # We have a list of objects listXSDataInput = self.__dictXSDataInput[ strKey ] for iIndex, xsDataInput in enumerate(listXSDataInput): strPathDataInput = "%s_%s_%d_dataInput.xml" % (strBasename, strKey, iIndex) EDUtilsFile.writeFile(strPathDataInput, xsDataInput.marshal()) def writeDataOutput(self, _edObject=None): """ Writes the output data object(s) into a working dir xml file """ self.DEBUG("EDPlugin.writeDataOutput") for strKey in self.__dictXSDataOutput.keys(): if (strKey == self.__strDefaultOutputDataKey): # "Old" style xsDataOutput = self.__dictXSDataOutput[ self.__strDefaultOutputDataKey ] if (xsDataOutput is not None): self.strPathDataOutput = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()) + "_dataOutput.xml") EDUtilsFile.writeFile(self.strPathDataOutput, xsDataOutput.marshal()) else: listXSDataOutput = self.__dictXSDataOutput[ strKey ] for iIndex, xsDataOutput in enumerate(listXSDataOutput): if (xsDataOutput is not None): strPathDataOutput = os.path.join(self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()) + "_" + strKey + "_%d_dataOutput.xml" % iIndex) EDUtilsFile.writeFile(strPathDataOutput, xsDataOutput.marshal()) def getBaseName(self): """ Returns the plugin base name """ if (self.__strBaseName is None): self.__strBaseName = self.createBaseName() return self.__strBaseName def setBaseName(self, _strBaseName): """ Sets the plugin base name """ self.__strBaseName = self.compactPluginName(_strBaseName) self.setName(self.__strBaseName) # Create the directory baseDirectory/baseName which will be used as working directory strWorkingDirPath = os.path.join(self.getBaseDirectory(), self.__strBaseName) if not os.path.isdir(strWorkingDirPath): os.mkdir(strWorkingDirPath) self.setWorkingDirectory(strWorkingDirPath) def createBaseName(self): """ Generates the plugin base name: (<prefix>-<object ID>) """ # First try to use global instance ID from EDObject strBaseName = "%s-%08d" % (self.compactPluginName(self.getPluginName()), self.getId()) strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName) # Try to create the directory... try: os.mkdir(strBaseDir) except Exception as strErrorDetail: self.error("EDPlugin.createBaseName: Could not create base directory %s because of %s" % (strBaseDir, strErrorDetail)) self.warning("EDPlugin.createBaseName: Trying to create alternative base directory...") self.writeErrorTrace() strTempDir = tempfile.mkdtemp(prefix=strBaseName, dir=self.getBaseDirectory()) os.chmod(strTempDir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) strBaseName = os.path.split(strTempDir)[1] strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName) self.warning("EDPlugin.createBaseName: Alternative base directory created: %s" % strBaseDir) self.DEBUG("EDPlugin.createBaseName : Directory created = " + strBaseDir) return strBaseName def compactPluginName(self, _pluginName): """ The prefix is constructed from the plugin name with the following renaming: EDPlugin -> "" EDPluginExec -> "" EDPluginControl -> "Control" """ strCompactName = _pluginName if (strCompactName.startswith("EDPluginExec")): strCompactName = strCompactName[12:] elif (strCompactName.startswith("EDPluginControl")): strCompactName = strCompactName[8:] elif (strCompactName.startswith("EDPlugin")): strCompactName = strCompactName[8:] return strCompactName def setBaseDirectory(self, _strBaseDirectory): """ Sets the plugin base directory """ self.DEBUG("EDPlugin.setBaseDirectory : " + _strBaseDirectory) if (not os.path.isdir(_strBaseDirectory)): self.DEBUG("EDPlugin.setBaseDirectory: base directory %s does dot yet exist! Creating it." % _strBaseDirectory) os.mkdir(_strBaseDirectory) self.__strBaseDirectory = _strBaseDirectory def getBaseDirectory(self): """ Returns the plugin base directory """ self.DEBUG("EDPlugin.getBaseDirectory : %s" % self.__strBaseDirectory) if (self.__strBaseDirectory is None): self.__strBaseDirectory = os.getcwd() return self.__strBaseDirectory def setWorkingDirectory(self, _strWorkingDirectory): """ Sets the plugin working directory """ self.DEBUG("EDPlugin.setWorkingDirectory : " + _strWorkingDirectory) self.__strWorkingDirectory = _strWorkingDirectory if not os.path.isdir(_strWorkingDirectory): self.DEBUG("EDPlugin.setWorkingDirectory, creating working directory %s." % _strWorkingDirectory) os.mkdir(self.__strWorkingDirectory) def getWorkingDirectory(self): """ Returns the plugin base directory """ self.DEBUG("EDPlugin.getWorkingDirectory : %s" % self.__strWorkingDirectory) returnValue = None if (self.__strWorkingDirectory is not None): returnValue = self.__strWorkingDirectory return returnValue def checkMandatoryParameters(self, _xsData, _strParamName): """ Checks that a mandatory parameter exists in the data If not, an error message is added in the list and the plugin fails """ if _xsData is None or (hasattr(_xsData, '__len__') and len(_xsData) == 0): strErrorMessage = "%s: input parameter is missing: %s" % (self.getPluginName(), _strParamName) self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError(strErrorMessage) def checkImportantParameters(self, _xsData, _strParamName): """ Checks that a specific parameter exists in the data If not, a warning message is added in the list """ if(_xsData == None): strWarningMessage = "%s: input parameter is missing: %s" % (self.getPluginName(), _strParamName) self.warning(strWarningMessage) self.addWarningMessage(strWarningMessage) def addExecutiveSummaryLine(self, _strExecutiveSummaryLine): """ Add a line to the executive summary string. """ self.DEBUG("EDPlugin.addExecutiveSummaryLine : %r" % _strExecutiveSummaryLine) strExecutiveSummaryLine = _strExecutiveSummaryLine if (not strExecutiveSummaryLine == ""): if (strExecutiveSummaryLine[-1] == "\n"): strExecutiveSummaryLine = strExecutiveSummaryLine[:-1] self.__listExecutiveSummaryLines.append(strExecutiveSummaryLine) def addExecutiveSummarySeparator(self, _strSeparator=None): """ Adds a separator to split the executive summary into different parts Default is a dotted line """ strSeparator = _strSeparator if (strSeparator is None): strSeparator = self.__strExecutiveSummarySeparator # Check that the last line doesn't already contain a separator if self.__listExecutiveSummaryLines != []: if self.__listExecutiveSummaryLines[-1] != strSeparator: self.addExecutiveSummaryLine(strSeparator) else: self.addExecutiveSummaryLine(strSeparator) def getListExecutiveSummaryLines(self): """ Returns the executive summary (list of text lines) """ return self.__listExecutiveSummaryLines def verboseScreenExecutiveSummary(self): """ Prints the executive summary on the screen """ for line in self.getListExecutiveSummaryLines(): self.screen(line) def verboseDebug(self, _strMessage): self.DEBUG(self.getPluginName() + " : " + _strMessage) def getPluginName(self): return self.getClassName() def getListOfDataInputKeys(self): return self.__dictXSDataInput.keys() def getListOfDataOutputKeys(self): return self.__dictXSDataOutput.keys() def getDefaultInputDataKey(self): return self.__strDefaultInputDataKey def getDefaultOutputDataKey(self): return self.__strDefaultOutputDataKey def getExecutiveSummarySeparator(self): return self.__strExecutiveSummarySeparator def isRequiredToHaveConfiguration(self): """ If the return value from this method is true, the plugin is required to have a configuration in order to be executed in a plugin execution test case. @return: RequiredToHaveConfiguration @rtype: boolean """ return self.__isRequiredToHaveConfiguration def setRequiredToHaveConfiguration(self, _bValue=True): """ Sets or unsets the plugin to be required to have a configuration for execution in a plugin execution test case. plugin execution test case. @param _bValue: RequiredToHaveConfiguration @type: boolean """ self.__isRequiredToHaveConfiguration = _bValue def setWriteXMLInputOutput(self, _bValue=True): """ Sets or unsets the plugin to write XML input and output files. @param _bValue: WriteDataXMLInputOutput @type: boolean """ self.__bWriteDataXMLInputOutput = _bValue def setWriteXMLInput(self, _bValue=True): """ Sets or unsets the plugin to write XML input files. @param _bValue: WriteDataXMLInput @type: boolean """ self.__bWriteDataXMLInput = _bValue def setWriteXMLOutput(self, _bValue=True): """ Sets or unsets the plugin to write XML output files. @param _bValue: WriteDataXMLOutput @type: boolean """ self.__bWriteDataXMLOutput = _bValue def setUseWarningInsteadOfError(self, _bValue=True): """ Sets or unsets the plugin to use warning messages also for error messages. @param _bValue: UseWarningInsteadOfError @type: boolean """ self.__bUseWarningInsteadOfError = _bValue def error(self, _strErrorMessage): """ Overloaded from EDLogging. If self.__bUseWarningInsteadOfError is True a warning message is issued instead of an error message. """ if self.__bUseWarningInsteadOfError: self.warning(_strErrorMessage) else: EDAction.error(self, _strErrorMessage) def ERROR(self, _strErrorMessage): """ Uses the overloaded self.error method above. """ self.error(_strErrorMessage)
class EDJob(EDLogging): """ Create a module called EDJob * Most of what was done up to 09-2010 in EDParallelExecute should be done here * Each instance will be a job * Constructor takes a plugin name * Each instance will have taking an "setDataInput" method getting an XMLin (as string) * Each instance will gave a "getDataOutput" method with optional join * there could be a "join" method, waiting for the job to finish * Each instance will have a "execute" method and returning a JobId * Each instance will have a "setCallBack" method that stores the name of the external callback * provide status of a job * keep track of all plugin status * leave the time to plugin to initialize * static class retrieve job-instance, status, small-log ... * prevent multiple run of a single job ? * does not manage workload of the computer, should be managed at the ExecPlugin level Used for the tango binding, EDParallelExecute ... == class variables == dictPluginStatus[pluginName] = ["uninitialized"|"running"|"executed"| "failed"] dictJobs [JobId] = EDJob.Instance == static methods == getJob(JobId) """ PLUGIN_STATE_UNITIALIZED = "uninitialized" PLUGIN_STATE_RUNNING = "running" PLUGIN_STATE_SUCCESS = "success" PLUGIN_STATE_FAILURE = "failure" __edFactoryPlugin = EDFactoryPlugin() __dictJobs = {} __semaphore = Semaphore() __fStartTime = time.time() def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDLogging.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__pathXSDInput = None self.__pathXSDOutput = None self.__bXmlInputSet = False self.__status = None self.__name = None self.__runtime = None self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName) if self.__edPlugin is None: raise RuntimeError("Unable to create plugin %s" % self.__strPluginName) self.__jobId = "%s-%08i" % (self.__strPluginName, self.__edPlugin.getId()) with self.__class__.__semaphore: self.__class__.__dictJobs[self.__jobId] = self if (self.__edPlugin is None): self.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the job (plugin) input data. @param: _oDataInput: could be either an String XML or an XSData object. @param _strDataInputKey: the key of an input data dictionnary The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ if _oDataInput in ["", None]: self.__bXmlInputSet = False return else: with self.locked(): if (self.__edPlugin is not None): self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey) self.__bXmlInputSet = True else: self.WARNING("Setting DataInput for uninstanciated plugin %s." % self.__strPluginName) def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ if (self.__edPlugin is not None): return self.__edPlugin.getDataInput(_strDataInputKey).marshal() elif (self.__pathXSDInput is not None): return open(self.__pathXSDInput).read() else: self.WARNING("Getting DataInput for uninstanciated plugin %s." % self.__strPluginName) dataInput = property(getDataInput, setDataInput) def getDataOutput(self, _strDataOutputKey=None, _bWait=True): """ Returns the Plugin Output Data @param _bWait: shall we wait for the plugin to finish to retrieve output data: Yes by default. @type _bWait: boolean """ if _bWait: #Wait for plugin to finish befor returning data output self.synchronize() if (self.__edPlugin is not None): return self.__edPlugin.getDataOutput(_strDataOutputKey).marshal() elif self.__pathXSDOutput is not None: return open(self.__pathXSDOutput).read() else: self.WARNING("Getting DataOutput for uninstanciated plugin or plugin has been garbage collected or output data were not saved. JobID was %s ." % self.__jobId) dataOutput = property(getDataOutput) def execute(self): """ Launch the EDNA plugin @return: JobId @rtype: string """ if not self.__bXmlInputSet: self.WARNING("Not executing job %s as input is empty" % self.__jobId) if (self.__edPlugin is not None): with self.locked(): self.__edPlugin.connectSUCCESS(self.successPluginExecution) self.__edPlugin.connectFAILURE(self.failurePluginExecution) self.__status = EDJob.PLUGIN_STATE_RUNNING self.__edPlugin.execute() return self.__jobId else: self.WARNING("Trying to run a plugin that does not exist: %s " % self.__strPluginName) def synchronize(self): """ Synchronize the execution of the job with the calling thread. """ with self.locked(): strStatus = self.__status if strStatus == EDJob.PLUGIN_STATE_RUNNING: self.__edPlugin.synchronize() elif strStatus == EDJob.PLUGIN_STATE_UNITIALIZED: self.WARNING("Unable to synchronize %s jobs" % strStatus) else: self.DEBUG("Unable to synchronize %s jobs" % strStatus) @classmethod def synchronizeAll(cls): """ Wait for all jobs to finish. """ EDVerbose.DEBUG("EDJob.synchronizeAll class method ") listJob = cls.__dictJobs.keys() for jobid in listJob: job = cls.__dictJobs[jobid] job.synchronize() if len(cls.__dictJobs) != len(listJob): EDVerbose.WARNING("EDJob.synchronizeAll: New jobs have been launched while synchronizing") def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ with self.locked(): self.__status = EDJob.PLUGIN_STATE_SUCCESS self.screen("Plugin %s: success after %.3fs" % (self.__jobId, _edObject.getRunTime())) try: self.__edSlotSUCCESS.call(self.__jobId) except Exception: self.ERROR("Error in execution of Success call-back for %s" % self.__jobId) self.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: self.ERROR("Error in execution of Common call-back (after success) for %s" % self.__jobId) self.writeErrorTrace() def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ with self.locked(): self.__status = EDJob.PLUGIN_STATE_FAILURE self.screen("Plugin %s: failure after %.3fs" % (self.__jobId, _edObject.getRunTime())) try: self.__edSlotFAILURE.call(self.__jobId) except Exception: self.ERROR("Error in execution of Failure call-back for %s" % self.__jobId) self.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: self.ERROR("Error in execution of Common call-back (after failure) for %s" % self.__jobId) self.writeErrorTrace() def connectSUCCESS(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotSUCCESS.connect(_oMethod) def connectFAILURE(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotFAILURE.connect(_oMethod) def connectCallBack(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotCallBack.connect(_oMethod) def getJobId(self): """ @return: JobId i.e. EDPluginName-Number @rtype: string """ return self.__jobId jobId = property(getJobId, "EDJob.jobId: read-only property") getJobID = getJobId def getPluginName(self): """ @return: Name of the plugin @rtype: string """ return self.__strPluginName pluginName = property(getPluginName, "EDJob.pluginName: read-only property") def getPlugin(self): """ @return: the plugin (instance) @rtype: python object """ return self.__edPlugin plugin = property(getPlugin, "EDJob.plugin: read-only property") def getStatus(self): """ @return: status of the Job @rtype: string """ return self.__status status = property(getStatus, "EDJob.status: read-only property") def getName(self): return self.__name def setName(self, _strName): if self.__name is None: self.__name = _strName else: self.WARNING("EDJob.setName: One cannot rename a Job !!!") name = property(getName, setName, "EDJob.name: nickname of the job") def getMemSize(self): """ try to guess the size in memory of a job @return: expected size in memory """ if asizeof is not None: return asizeof.asizeof(self) @classmethod def getStatusFromID(cls, jobId): """ Retrieve the job (hence the plugin) status @param jobId: the Job identification number @type jobId: string @return: the EDJob status @rtype: string """ if jobId in cls.__dictJobs: strRet = cls.__dictJobs[jobId].getStatus() else: strRet = "Unable to retrieve such job: %s" % jobId EDVerbose.WARNING(strRet) return strRet getStatusFromId = getStatusFromID @classmethod def getJobFromID(cls, jobId): """ Retrieve the job (hence the plugin) @param jobId: the Job identification number @type jobId: string @return: the "EDJob instance", which contains the plugin (__edPlugin) and the status @rtype: a Python object. """ if jobId in cls.__dictJobs: return cls.__dictJobs[jobId] else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) getJobFromId = getJobFromID @classmethod def getMemoryFootprint(cls): if asizeof is not None: return asizeof.asizesof(cls.__dictJobs) def cleanJob(self, forceGC=True): """ Frees the memory associated with the top level plugin @param forceGC: Force garbage collection after clean-up @type forceGC: boolean """ self.synchronize() with self.locked(): if self.__edPlugin is not None: self.__pathXSDOutput = self.__edPlugin.strPathDataOutput self.__pathXSDInput = self.__edPlugin.strPathDataInput self.__runtime = self.__edPlugin.getRunTime() self.__edPlugin = None if forceGC: gc.collect() @classmethod def cleanJobfromId(cls, jobId, forceGC=True): """ Frees the memory associated with the top level plugin @param jobId: the Job identification number @type jobId: string @param forceGC: Force garbage collection after clean-up @type forceGC: boolean """ if jobId in cls.__dictJobs: job = cls.__dictJobs[jobId] job.cleanJob(forceGC) strRet = "Job %s cleaned" % jobId else: strRet = "Unable to retrieve such EDJob: %s" % jobId EDVerbose.WARNING(strRet) return strRet cleanJobfromID = cleanJobfromId @classmethod def getDataOutputFromId(cls, jobId): """ Returns the Plugin Output Data @param jobId: job idenfier @type jobId: string @return: edJob.DataOutput XML string """ output = None job = cls.getJobFromId(jobId) if job is not None: output = job.getDataOutput() return output or "" getDataOutputFromID = getDataOutputFromId @classmethod def getDataInputFromId(cls, jobId): """ Returns the Plugin Input Data @param jobId: job idenfier @type jobId: string @return: edJob.DataInput XML string """ output = None job = cls.getJobFromId(jobId) if job is not None: output = job.getDataInput() return output or "" getDataInputFromID = getDataInputFromId @classmethod def stats(cls): """ Retrieve some statistics and print them """ lstStrOut = [] output = [] fExecTime = time.time() - cls.__fStartTime keys = cls.__dictJobs.keys() keys.sort() for num, key in enumerate(keys) : job = cls.__dictJobs[key] if job.getPlugin() is None: runtime = job.__runtime else: runtime = job.getPlugin().getRunTime() output.append([num, key, job.getStatus(), runtime, job.getMemSize()]) output.sort() iNbJob = max(1, len(keys)) lstStrOut.append("_" * 110) lstStrOut.append("%s\t|\t%s\t\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("nr", "EDPluginName-Id", "status", "runtime", "memory")) lstStrOut.append("_" * 110) fWall = 0.0 fSumProd = 0.0 fSumX = 0.0 fSumX2 = 0.0 for oneJob in output: fWall += oneJob[3] fSumX += oneJob[0] fSumX2 += oneJob[0] * oneJob[0] fSumProd += oneJob[0] * oneJob[3] lstStrOut.append("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob)) lstStrOut.append("_" * 110) lstStrOut.append("Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime)) lstStrOut.append("Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob)) if len(keys) > 1: fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX) fOrd = (fWall - fSlope * fSumX) / iNbJob else: fSlope = 0.0 fOrd = fWall lstStrOut.append("Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope)) strOutput = os.linesep.join(lstStrOut) EDVerbose.screen(strOutput) return strOutput
class EDAction(EDLogging, Thread): """ This class is taking care of executing the EDNA plugin and application workflow preProcess - process - postProcess. The detailed workflow looks like this: preProcess if not failure: slotPreProcess if not failure: process if not failure: slotProcess if not failure: postProcess if not failure: slotPostProcess if not failure: slotSUCCESS if failure: slotFAILURE Always: finally Process """ def __init__(self): EDLogging.__init__(self) Thread.__init__(self) self.__edSlotPreProcess = EDSlot() self.__edSlotProcess = EDSlot() self.__edSlotPostProcess = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__edSlotFinallyProcess = EDSlot() self.__bIsFailure = False self.__bIsTimeOut = False self.__fTimeOutInSeconds = None self.__fDefaultTimeOutInSeconds = 600.0 self.__bIsAbort = False # Reference to the object which calls execute or executeSynchronous self.__edObject = None self.__lExtraTime = [] # list of extra allowed time for execution (in second) self.__bLogTiming = False def executeKernel(self): dictTimeStamps = { "init": time.time() } self.setTimeInit() try: if (not self.isFailure()): self.DEBUG("EDAction.executeKernel preProcess " + self.getClassName()) self.preProcess() if self.__bLogTiming: dictTimeStamps["preProcess"] = time.time() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel slotPreProcess " + self.getClassName()) self.__edSlotPreProcess.call(self) if self.__bLogTiming: dictTimeStamps["slotPreProcess"] = time.time() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel process " + self.getClassName()) self.process() if self.__bLogTiming: dictTimeStamps["process"] = time.time() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel slotProcess " + self.getClassName()) self.__edSlotProcess.call(self) if self.__bLogTiming: dictTimeStamps["slotProcess"] = time.time() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel postProcess " + self.getClassName()) self.postProcess() if self.__bLogTiming: dictTimeStamps["postProcess"] = time.time() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel slotPostProcess " + self.getClassName()) self.__edSlotPostProcess.call(self) if self.__bLogTiming: dictTimeStamps["slotPostProcess"] = time.time() except Exception: self.writeErrorTrace() self.setFailure() # Execute finally process even in case of failure self.DEBUG("EDAction.executeKernel finallyProcess" + self.getClassName()) try: self.finallyProcess() if self.__bLogTiming: dictTimeStamps["finallyProcess"] = time.time() except Exception: self.DEBUG("EDAction.executeKernel: ERROR in finallyProcess!") self.writeErrorTrace() self.setFailure() try: self.__edSlotFinallyProcess.call(self) except Exception: self.DEBUG("EDAction.executeKernel: ERROR in slotFinallyProcess!") self.writeErrorTrace() self.setFailure() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel slotSUCCESS") # Check that something doesn't go wrong in the success method! try: self.__edSlotSUCCESS.call(self) if self.__bLogTiming: dictTimeStamps["slotSUCCESS"] = time.time() except Exception: self.DEBUG("EDAction.executeKernel: ERROR in slotSUCCESS!") self.writeErrorTrace() self.setFailure() if (self.isFailure()): self.DEBUG("EDAction.executeKernel slotFAILURE") # Check that something doesn't go wrong in the success method! try: self.__edSlotFAILURE.call(self) if self.__bLogTiming: dictTimeStamps["slotFAILURE"] = time.time() except Exception: self.DEBUG("EDAction.executeKernel: ERROR in slotFAILURE!") self.writeErrorTrace() self.setTimeEnd() if self.__bLogTiming: lstTimings = [] dictTimeStamps["end"] = time.time() lstTimings.append("EDAction.executeKernel: profiling of %s %i \t total time duration = %.3f s" % (self.getClassName(), self.getId(), dictTimeStamps["end"] - dictTimeStamps["init"])) fTimeForFailureCalculation = dictTimeStamps["init"] if "preProcess" in dictTimeStamps: lstTimings.append("\t preProcess \t\t time duration = %.3f s" % (dictTimeStamps["preProcess"] - dictTimeStamps["init"])) fTimeForFailureCalculation = dictTimeStamps["preProcess"] if "process" in dictTimeStamps: lstTimings.append("\t process \t\t time duration = %.3f s" % (dictTimeStamps["process"] - dictTimeStamps["slotPreProcess"])) fTimeForFailureCalculation = dictTimeStamps["process"] if "postProcess" in dictTimeStamps: lstTimings.append("\t postProcess \t\t time duration = %.3f s" % (dictTimeStamps["postProcess"] - dictTimeStamps["slotProcess"])) fTimeForFailureCalculation = dictTimeStamps["postProcess"] fTimeForFinallyCalculation = fTimeForFailureCalculation if "slotSUCCESS" in dictTimeStamps: lstTimings.append("\t slotSUCCESS \t\t time duration = %.3f s" % (dictTimeStamps["slotSUCCESS"] - dictTimeStamps["slotPostProcess"])) fTimeForFailureCalculation = dictTimeStamps["slotSUCCESS"] if "slotFAILURE" in dictTimeStamps: lstTimings.append("\t slotFAILURE \t\t time duration = %.3f s" % (dictTimeStamps["slotFAILURE"] - fTimeForFailureCalculation)) if dictTimeStamps.has_key("finallyProcess"): lstTimings.append("\t finallyProcess \t time duration = %.3f s" % (dictTimeStamps["finallyProcess"] - fTimeForFinallyCalculation)) self.log(os.linesep.join(lstTimings)) def synchronize(self): """ Wait for the thread to finish. Since the time out is used by e.g. EDPluginExecProcessScript we add an extra second in order to allow the subclasses to handle the time out - without this extra second it's the EDAction class who time-outs first. Note that this does not in any case add time to the execution, because the extra second is only used for time-outs. The method returns immediately once the thread has finished. """ EDVerbose.DEBUG("EDAction.synchronize() for %s" % self.getName()) # fTimeOut = self.__fTimeOutInSeconds if self.__fTimeOutInSeconds is None: self.__fTimeOutInSeconds = self.__fDefaultTimeOutInSeconds #Wait for the thread to be started up to timeout if self.isStarted() == False: tStartWait = time.time() while self.isStarted() == False: time.sleep(1) if time.time() - tStartWait > self.__fTimeOutInSeconds: self.__bIsTimeOut = True strErrorMessage = "Timeout when waiting for %s to start!" % self.getClassName() EDVerbose.DEBUG("EDAction.synchronize: " + strErrorMessage) EDVerbose.ERROR(strErrorMessage) self.setFailure() return # We add an extra second in order to allow execution plugin to finish # which have the same timeout self.join(float(self.__fTimeOutInSeconds + 1)) for fExtraTime in self.__lExtraTime: self.join(float(fExtraTime)) if self.isAlive(): # Timeout! self.__bIsTimeOut = True EDVerbose.DEBUG("EDAction.synchronize: Timeout!") strErrorMessage = "Timeout when waiting for %s to terminate." % self.getClassName() EDVerbose.ERROR(strErrorMessage) self.setFailure() def isTimeOut(self): return self.__bIsTimeOut def hasTimedOut(self, _bTimeout): """ Enforce the timeout state @param _bTimeout: if you think you can force this ! @type _bTimeout: boolean """ self.__bIsTimeOut = bool(_bTimeout) def isFailure(self): return self.__bIsFailure def setFailure(self): self.__bIsFailure = True def run(self): self.executeKernel() def executeAction(self, _edObject=None): self.execute(_edObject) def executeActionSynchronous(self, _edObject=None): self.executeSynchronous() def connectPreProcess(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotPreProcess.connect(_oMethod) self.synchronizeOff() def connectProcess(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotProcess.connect(_oMethod) self.synchronizeOff() def connectPostProcess(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotPostProcess.connect(_oMethod) self.synchronizeOff() def connectSUCCESS(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotSUCCESS.connect(_oMethod) self.synchronizeOff() def connectFAILURE(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotFAILURE.connect(_oMethod) self.synchronizeOff() def connectFinallyProcess(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotFinallyProcess.connect(_oMethod) self.synchronizeOff() def isRunning(self): return self.isAlive() def isEnded(self): EDVerbose.DEBUG("%s.isEnded return %s" % (self.getName(), (self.getTimeEnd() is not None))) return (self.getTimeEnd() is not None) def isStarted(self): EDVerbose.DEBUG("%s.isStarted return %s, %s " % (self.getName(), (self.getTimeInit() is not None), self.getTimeInit())) return (self.getTimeInit() is not None) def setTimeOut(self, _fTimeOut): """ Sets the time out """ EDVerbose.DEBUG("EDAction.setTimeOut called with value %s" % _fTimeOut) self.__fTimeOutInSeconds = float(_fTimeOut) def getTimeOut(self): """ Returns the time out """ if self.__fTimeOutInSeconds is None: self.__fTimeOutInSeconds = self.__fDefaultTimeOutInSeconds return self.__fTimeOutInSeconds def addExtraTime(self, _fExtraTime): """ Allows to incread the timeout of a plugin while it is running @param _fExtraTime: extra time to be added to timeout @type _fExtraTime: float """ self.__lExtraTime.append(_fExtraTime) self.setTimeOut(self.getTimeOut() + _fExtraTime) def getDefaultTimeOut(self): """ Returns the time out """ return self.__fDefaultTimeOutInSeconds def getSlotSUCCESS(self): return self.__edSlotSUCCESS def getSlotFAILURE(self): return self.__edSlotFAILURE def preProcess(self, _edObject=None): pass def process(self, _edObject=None): pass def postProcess(self, _edObject=None): pass def abort(self, _edObject=None): pass def finallyProcess(self, _edObject=None): pass def execute(self, _edObject=None): self.__bIsStarted = True self.__edObject = _edObject self.start() def executeSynchronous(self, _edObject=None): self.execute(_edObject) self.synchronize() def setLogTiming(self, _bValue): """ Force this action to log it's timing to file """ self.__bLogTiming = bool(_bValue) def getLogTiming(self): return self.__bLogTiming logTiming = property(getLogTiming, setLogTiming)
class EDAction(EDLogging, Thread): """ This class is taking care of executing the EDNA plugin and application workflow preProcess - process - postProcess. The detailed workflow looks like this: preProcess if not failure: slotPreProcess if not failure: process if not failure: slotProcess if not failure: postProcess if not failure: slotPostProcess if not failure: slotSUCCESS if failure: slotFAILURE Always: finally Process """ def __init__(self): EDLogging.__init__(self) Thread.__init__(self) self.__edSlotPreProcess = EDSlot() self.__edSlotProcess = EDSlot() self.__edSlotPostProcess = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__edSlotFinallyProcess = EDSlot() self.__bIsFailure = False self.__bIsTimeOut = False self.__fTimeOutInSeconds = None self.__fDefaultTimeOutInSeconds = 600.0 self.__bIsAbort = False # Reference to the object which calls execute or executeSynchronous self.__edObject = None self.__lExtraTime = [ ] # list of extra allowed time for execution (in second) self.__bLogTiming = False def executeKernel(self): dictTimeStamps = {"init": time.time()} self.setTimeInit() try: if (not self.isFailure()): self.DEBUG("EDAction.executeKernel preProcess " + self.getClassName()) self.preProcess() if self.__bLogTiming: dictTimeStamps["preProcess"] = time.time() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel slotPreProcess " + self.getClassName()) self.__edSlotPreProcess.call(self) if self.__bLogTiming: dictTimeStamps["slotPreProcess"] = time.time() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel process " + self.getClassName()) self.process() if self.__bLogTiming: dictTimeStamps["process"] = time.time() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel slotProcess " + self.getClassName()) self.__edSlotProcess.call(self) if self.__bLogTiming: dictTimeStamps["slotProcess"] = time.time() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel postProcess " + self.getClassName()) self.postProcess() if self.__bLogTiming: dictTimeStamps["postProcess"] = time.time() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel slotPostProcess " + self.getClassName()) self.__edSlotPostProcess.call(self) if self.__bLogTiming: dictTimeStamps["slotPostProcess"] = time.time() except Exception: self.writeErrorTrace() self.setFailure() # Execute finally process even in case of failure self.DEBUG("EDAction.executeKernel finallyProcess" + self.getClassName()) try: self.finallyProcess() if self.__bLogTiming: dictTimeStamps["finallyProcess"] = time.time() except Exception: self.DEBUG("EDAction.executeKernel: ERROR in finallyProcess!") self.writeErrorTrace() self.setFailure() try: self.__edSlotFinallyProcess.call(self) except Exception: self.DEBUG("EDAction.executeKernel: ERROR in slotFinallyProcess!") self.writeErrorTrace() self.setFailure() if (not self.isFailure()): self.DEBUG("EDAction.executeKernel slotSUCCESS") # Check that something doesn't go wrong in the success method! try: self.__edSlotSUCCESS.call(self) if self.__bLogTiming: dictTimeStamps["slotSUCCESS"] = time.time() except Exception: self.DEBUG("EDAction.executeKernel: ERROR in slotSUCCESS!") self.writeErrorTrace() self.setFailure() if (self.isFailure()): self.DEBUG("EDAction.executeKernel slotFAILURE") # Check that something doesn't go wrong in the success method! try: self.__edSlotFAILURE.call(self) if self.__bLogTiming: dictTimeStamps["slotFAILURE"] = time.time() except Exception: self.DEBUG("EDAction.executeKernel: ERROR in slotFAILURE!") self.writeErrorTrace() self.setTimeEnd() if self.__bLogTiming: lstTimings = [] dictTimeStamps["end"] = time.time() lstTimings.append( "EDAction.executeKernel: profiling of %s %i \t total time duration = %.3f s" % (self.getClassName(), self.getId(), dictTimeStamps["end"] - dictTimeStamps["init"])) fTimeForFailureCalculation = dictTimeStamps["init"] if "preProcess" in dictTimeStamps: lstTimings.append( "\t preProcess \t\t time duration = %.3f s" % (dictTimeStamps["preProcess"] - dictTimeStamps["init"])) fTimeForFailureCalculation = dictTimeStamps["preProcess"] if "process" in dictTimeStamps: lstTimings.append("\t process \t\t time duration = %.3f s" % (dictTimeStamps["process"] - dictTimeStamps["slotPreProcess"])) fTimeForFailureCalculation = dictTimeStamps["process"] if "postProcess" in dictTimeStamps: lstTimings.append( "\t postProcess \t\t time duration = %.3f s" % (dictTimeStamps["postProcess"] - dictTimeStamps["slotProcess"])) fTimeForFailureCalculation = dictTimeStamps["postProcess"] fTimeForFinallyCalculation = fTimeForFailureCalculation if "slotSUCCESS" in dictTimeStamps: lstTimings.append( "\t slotSUCCESS \t\t time duration = %.3f s" % (dictTimeStamps["slotSUCCESS"] - dictTimeStamps["slotPostProcess"])) fTimeForFailureCalculation = dictTimeStamps["slotSUCCESS"] if "slotFAILURE" in dictTimeStamps: lstTimings.append( "\t slotFAILURE \t\t time duration = %.3f s" % (dictTimeStamps["slotFAILURE"] - fTimeForFailureCalculation)) if "finallyProcess" in dictTimeStamps: lstTimings.append( "\t finallyProcess \t time duration = %.3f s" % (dictTimeStamps["finallyProcess"] - fTimeForFinallyCalculation)) self.log(os.linesep.join(lstTimings)) def synchronize(self): """ Wait for the thread to finish. Since the time out is used by e.g. EDPluginExecProcessScript we add an extra second in order to allow the subclasses to handle the time out - without this extra second it's the EDAction class who time-outs first. Note that this does not in any case add time to the execution, because the extra second is only used for time-outs. The method returns immediately once the thread has finished. """ EDVerbose.DEBUG("EDAction.synchronize() for %s" % self.getName()) # fTimeOut = self.__fTimeOutInSeconds if self.__fTimeOutInSeconds is None: self.__fTimeOutInSeconds = self.__fDefaultTimeOutInSeconds #Wait for the thread to be started up to timeout if self.isStarted() == False: tStartWait = time.time() while self.isStarted() == False: time.sleep(1) if time.time() - tStartWait > self.__fTimeOutInSeconds: self.__bIsTimeOut = True strErrorMessage = "Timeout when waiting for %s to start!" % self.getClassName( ) EDVerbose.DEBUG("EDAction.synchronize: " + strErrorMessage) EDVerbose.ERROR(strErrorMessage) self.setFailure() return # We add an extra second in order to allow execution plugin to finish # which have the same timeout self.join(float(self.__fTimeOutInSeconds + 1)) for fExtraTime in self.__lExtraTime: self.join(float(fExtraTime)) if self.isAlive(): # Timeout! self.__bIsTimeOut = True EDVerbose.DEBUG("EDAction.synchronize: Timeout!") strErrorMessage = "Timeout when waiting for %s to terminate." % self.getClassName( ) EDVerbose.ERROR(strErrorMessage) self.setFailure() def isTimeOut(self): return self.__bIsTimeOut def hasTimedOut(self, _bTimeout): """ Enforce the timeout state @param _bTimeout: if you think you can force this ! @type _bTimeout: boolean """ self.__bIsTimeOut = bool(_bTimeout) def isFailure(self): return self.__bIsFailure def setFailure(self, value=True): """ use setFailure(False) to unset the failure flag """ self.__bIsFailure = value def run(self): self.executeKernel() def executeAction(self, _edObject=None): self.execute(_edObject) def executeActionSynchronous(self, _edObject=None): self.executeSynchronous() def connectPreProcess(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotPreProcess.connect(_oMethod) self.synchronizeOff() def connectProcess(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotProcess.connect(_oMethod) self.synchronizeOff() def connectPostProcess(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotPostProcess.connect(_oMethod) self.synchronizeOff() def connectSUCCESS(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotSUCCESS.connect(_oMethod) self.synchronizeOff() def connectFAILURE(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotFAILURE.connect(_oMethod) self.synchronizeOff() def connectFinallyProcess(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotFinallyProcess.connect(_oMethod) self.synchronizeOff() def isRunning(self): return self.isAlive() def isEnded(self): EDVerbose.DEBUG("%s.isEnded return %s" % (self.getName(), (self.getTimeEnd() is not None))) return (self.getTimeEnd() is not None) def isStarted(self): EDVerbose.DEBUG("%s.isStarted return %s, %s " % (self.getName(), (self.getTimeInit() is not None), self.getTimeInit())) return (self.getTimeInit() is not None) def setTimeOut(self, _fTimeOut): """ Sets the time out """ EDVerbose.DEBUG("EDAction.setTimeOut called with value %s" % _fTimeOut) self.__fTimeOutInSeconds = float(_fTimeOut) def getTimeOut(self): """ Returns the time out """ if self.__fTimeOutInSeconds is None: self.__fTimeOutInSeconds = self.__fDefaultTimeOutInSeconds return self.__fTimeOutInSeconds def addExtraTime(self, _fExtraTime): """ Allows to incread the timeout of a plugin while it is running @param _fExtraTime: extra time to be added to timeout @type _fExtraTime: float """ self.__lExtraTime.append(_fExtraTime) self.setTimeOut(self.getTimeOut() + _fExtraTime) def getDefaultTimeOut(self): """ Returns the time out """ return self.__fDefaultTimeOutInSeconds def getSlotSUCCESS(self): return self.__edSlotSUCCESS def getSlotFAILURE(self): return self.__edSlotFAILURE def preProcess(self, _edObject=None): pass def process(self, _edObject=None): pass def postProcess(self, _edObject=None): pass def abort(self, _edObject=None): pass def finallyProcess(self, _edObject=None): pass def execute(self, _edObject=None): self.__bIsStarted = True self.__edObject = _edObject self.start() def executeSynchronous(self, _edObject=None): self.execute(_edObject) self.synchronize() def setLogTiming(self, _bValue): """ Force this action to log it's timing to file """ self.__bLogTiming = bool(_bValue) def getLogTiming(self): return self.__bLogTiming logTiming = property(getLogTiming, setLogTiming)
class EDPlugin(EDAction): """ This is the EDNA plugin main class An EDNA plugin class: - is a configurable entity - has a base name (<date>-<random number>-<base name>) - handles input/output data (setter, getter, checker) - has warning and error messages - has a base and a working directory (both are configurable) The working directory is the folder from which the plugin is launched and should contain all associated files with the plugin execution (edna xml input/output, 3rd party output files) The base directory is the parent directory of the working directory. Example: the working directory of a control plugin is the base directory of the plugins that it invokes, i.e. the plugins working directories has the control plugin working directory as parent. - defines the method that generates an executive summary (user-related output summary) that sub-classes should implement """ CONF_BASE_DIR_LABEL = "baseDirectory" CONF_WORKING_DIR_LABEL = "workingDirectory" CONF_TIME_OUT = "timeOut" CONF_WRITE_XML_INPUT_OUTPUT = "writeXMLInputOutput" CONF_WRITE_XML_OUTPUT = "writeXMLOutput" CONF_WRITE_XML_INPUT = "writeXMLInput" def __init__(self): """ Initializes plugin related attributes described above """ EDAction.__init__(self) self.__xsPluginItem = None self.__dictXSDataInputClass = {} self.__dictXSDataInput = {} self.__dictXSDataOutput = {} self.__strDefaultInputDataKey = "defaultInputData" self.__strDefaultOutputDataKey = "defaultOutputData" self.__edSlotExportDataOutput = EDSlot() self.__strBaseDirectory = None self.__strWorkingDirectory = None self.__strBaseName = None self.__listExecutiveSummaryLines = [] self.__strExecutiveSummarySeparator = "-" * 80 self.__listErrorMessages = [] self.__listWarningMessages = [] self.__isRequiredToHaveConfiguration = False self.__bWriteDataXMLInputOutput = True self.__bWriteDataXMLOutput = True self.__bWriteDataXMLInput = True self.__strPluginId = "%s-%08i" % (self.getClassName(), self.getId()) self.strPathDataInput = None self.strPathDataOutput = None self.__bUseWarningInsteadOfError = False self.__edConfiguration = EDConfigurationStatic() def preProcess(self, _edObject=None): """ Writes xml data input in the working dir (if required) Connects a slot for generating the executive summary after the plugin execution Connects a slot for checking output data to the finally process Initialize the base directory Configures the plugin Checks the input data """ EDAction.preProcess(self, _edObject) self.DEBUG("EDPlugin.preProcess") self.connectPostProcess(self.exportDataOutput) if self.__bWriteDataXMLInputOutput: if self.__bWriteDataXMLInput: self.connectPreProcess(self.writeDataInput) self.connectPostProcess(self.generateExecutiveSummary) self.connectFinallyProcess(self.checkDataOutput) if (self.__strBaseName is None): self.setBaseName(self.createBaseName()) EDStatus.tellRunning(self.__strPluginId) self.connectFinallyProcess(self.tellFinished) self.checkParameters() def tellFinished(self, _edObject=None): """ Tell EDStatus that the plugin has finished, either in success either in error """ if self.isFailure(): EDStatus.tellFailure(self.__strPluginId) else: EDStatus.tellSuccess(self.__strPluginId) def checkDataOutput(self, _edObject=None): """ Checks if output data is available, if not issues a warning and sets an empty XSDataResult as output data Writes xml data output in the working dir (if required) """ EDAction.finallyProcess(self, _edObject) if self.__dictXSDataOutput == {}: strWarningMessage = "Output data for plugin %s not set, using XSDataResult as output" % self.getPluginName( ) self.WARNING(strWarningMessage) self.addWarningMessage(strWarningMessage) self.setDataOutput(XSDataResult()) if self.__bWriteDataXMLInputOutput: if self.__bWriteDataXMLOutput: self.writeDataOutput() def synchronize(self): """ This method calls EDAction.synchronize and if a time-out occurs an error message is added to the list of error messages. """ EDAction.synchronize(self) if self.isTimeOut(): strErrorMessage = "Timeout when waiting for %s to terminate." % self.getClassName( ) self.addErrorMessage(strErrorMessage) def getConfig(self): """ Gets the Plugin Configuration as a dictionary """ # self.DEBUG("EDPlugin.getConfig") return self.__edConfiguration.get(self.getPluginName(), {}) def setConfig(self, _dict, _bLocal=False): """ Receives a dictionary (Plugin Configuration) from the application. """ self.DEBUG("EDPlugin.setConfiguration") if _bLocal: self.__edConfiguration = EDConfiguration() if _dict is not None: self.__edConfiguration[self.getPluginName()] = _dict else: self.__edConfiguration[self.getPluginName()] = {} config = property(getConfig, setConfig) def configure(self): """ Should be overridden by the Final Plugin If needed This method should set its proper members attributes from a Plugin configuration Object """ self.DEBUG("EDPlugin.configure : plugin name = %s, EDNA_SITE = %s" % (self.getPluginName(), EDUtilsPath.EDNA_SITE)) # set Timeout if different from default one if self.getTimeOut() == self.getDefaultTimeOut(): # Try to get time out from plugin configuration iTimeOut = self.config.get(EDPlugin.CONF_TIME_OUT, None) if iTimeOut is not None: self.DEBUG( "EDPlugin.configure: Setting time out to %d s from plugin configuration." % iTimeOut) self.setTimeOut(iTimeOut) else: self.DEBUG( "EDPlugin.configure: timeout already set before plugin is configured." ) # Base directory strBaseDirectory = self.getBaseDirectory() if (strBaseDirectory is None): # Try to get base directory from plugin configuration strBaseDirectory = self.config.get(EDPlugin.CONF_BASE_DIR_LABEL, None) if (strBaseDirectory is None): # Try to get working directory from environment variable strBaseDirectory = os.environ.get("EDNA_BASE_DIRECTORY") if (strBaseDirectory is None): self.DEBUG( "EDPlugin.configure: Using current base directory as working directory." ) strBaseDirectory = os.getcwd() else: self.DEBUG( "EDPlugin.configure: Setting base directory from $EDNA_WORKING_DIRECTORY." ) else: if (strBaseDirectory == "."): self.DEBUG( "EDPlugin.configure: Using current base directory as working directory." ) strBaseDirectory = os.getcwd() else: strBaseDirectory = os.path.abspath(strBaseDirectory) self.DEBUG( "EDPlugin.configure: Setting base directory from plugin configuration." ) self.setBaseDirectory(strBaseDirectory) else: self.DEBUG( "EDPlugin.configure: Base directory already set before plugin is configured." ) # Working directory strWorkingDirectory = self.getWorkingDirectory() if (strWorkingDirectory is None): # Try to get working directory from plugin configuration strWorkingDirectory = self.config.get( EDPlugin.CONF_WORKING_DIR_LABEL, None) if (strWorkingDirectory is not None): self.DEBUG( "EDPlugin.configure: Setting working directory from plugin configuration." ) else: self.DEBUG( "EDPlugin.configure: Setting working directory as base directory + base name." ) strWorkingDirectory = os.path.join(self.getBaseDirectory(), self.getBaseName()) self.setWorkingDirectory(strWorkingDirectory) else: self.DEBUG( "EDPlugin.configure: Working directory already set before plugin is configured." ) # self.__bWriteDataXMLInputOutput = bool( self.config.get(self.CONF_WRITE_XML_INPUT_OUTPUT, True)) self.__bWriteDataXMLOutput = bool( self.config.get(self.CONF_WRITE_XML_OUTPUT, True)) self.__bWriteDataXMLInput = bool( self.config.get(self.CONF_WRITE_XML_INPUT, True)) def execute(self, _edObject=None): # Configure the plugin before starting it's thread self.configure() EDAction.execute(self, _edObject) def checkParameters(self): """ Should be overridden by the Final Plugin If needed This method should check that the data input are consistent """ self.DEBUG("EDPlugin.checkParameters") def setXSDataInputClass(self, _xsDataInputClass, _strDataInputKey=None): """ This method should be called in the constructor of the derived plugins in order to set the XSData type of the input data, e.g. XSDataInputXXX """ strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInputClass.keys()): strErrorMessage = "ERROR: " + self.getPluginName( ) + ".setXSDataInputClass, Data Input Class already defined for key: " + strDataInputKey self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError(strErrorMessage) self.__dictXSDataInputClass[strDataInputKey] = _xsDataInputClass def getXSDataInputClass(self, _strDataInputKey=None): """ Returns the XSData type of the input data. """ pyXSDataInputClass = None strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInputClass.keys()): pyXSDataInputClass = self.__dictXSDataInputClass[strDataInputKey] return pyXSDataInputClass def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the plugin input data. _oDataInput could be either an String XML or an XSData object. The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ self.DEBUG("EDPlugin.setDataInput") strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey # Allow for None input if (_oDataInput is None): self.DEBUG("EDPlugin.setDataInput: Data input is None") self.__dictXSDataInput[strDataInputKey] = [] elif (self.getXSDataInputClass(strDataInputKey) is None): strErrorMessage = "ERROR: " + self.getPluginName( ) + ".setDataInput, Data Input Class not defined for key: " + strDataInputKey self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError(strErrorMessage) else: # Check the type xsDataInput = None if isinstance(_oDataInput, (str, unicode)): self.DEBUG("EDPlugin.setDataInput: Input Data is string ") xsDataInput = self.getXSDataInputClass( strDataInputKey).parseString(_oDataInput) elif (isinstance(_oDataInput, self.getXSDataInputClass(strDataInputKey))): self.DEBUG("EDPlugin.setDataInput: Input Data is of type " + str(_oDataInput.__class__)) xsDataInput = _oDataInput else: strErrorMessage = "ERROR: %s.setDataInput, wrong data type %r for data input key: %s, expected XML string or %r" % \ (self.getPluginName(), _oDataInput.__class__, strDataInputKey, self.getXSDataInputClass(strDataInputKey)) self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError(strErrorMessage) # Add the object to a list if its key is not the default key if (strDataInputKey != self.__strDefaultInputDataKey): # Check if there's already a list stored if (not strDataInputKey in self.__dictXSDataInput.keys()): self.__dictXSDataInput[strDataInputKey] = [] self.__dictXSDataInput[strDataInputKey].append(xsDataInput) else: self.__dictXSDataInput[strDataInputKey] = xsDataInput def hasDataInput(self, _strDataInputKey=None): """ Returns True if the plugin has Input Data for a particular key. If the key is not provided a default key is used. """ strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInput.keys()): return True else: return False def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ oValue = None strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInput.keys()): oValue = self.__dictXSDataInput[strDataInputKey] else: strErrorMessage = self.getPluginName( ) + ".getDataInput, no input data defined for key: " + strDataInputKey self.warning(strErrorMessage) self.addWarningMessage(strErrorMessage) return oValue def delDataInput(self, _strDataInputKey=None): """ Deletes the data input for a particular key. If the key is not provided a default key is used. """ strDataInputKey = _strDataInputKey if (strDataInputKey is None): strDataInputKey = self.__strDefaultInputDataKey if (strDataInputKey in self.__dictXSDataInput.keys()): self.__dictXSDataInput[strDataInputKey] = None else: strErrorMessage = self.getPluginName( ) + ".delDataInput, no input data defined for key: " + strDataInputKey self.warning(strErrorMessage) self.addWarningMessage(strErrorMessage) # Property for dataInput dataInput = property(getDataInput, setDataInput, delDataInput, "Property for dataInput") def setDataOutput(self, _xsDataOutput, _strDataOutputKey=None): """ Sets the plugin output data for a particular key. If the key is not provided a default key is used. If the key is already defined in the dictionary, the corresponding data object is added to a list which contains the already stored object(s). """ strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey # Add the object to a list if its key not the default key if (strDataOutputKey == self.__strDefaultOutputDataKey): self.__dictXSDataOutput[strDataOutputKey] = _xsDataOutput else: # Check if the _xsDataoutput object is already a list if (type(_xsDataOutput) == list): self.__dictXSDataOutput[strDataOutputKey] = _xsDataOutput else: # Check if the stored object contains already a list if (not strDataOutputKey in self.__dictXSDataOutput.keys()): self.__dictXSDataOutput[strDataOutputKey] = [] self.__dictXSDataOutput[strDataOutputKey].append(_xsDataOutput) def getDataOutput(self, _strDataOutputKey=None): """ Returns the Plugin Output Data """ oValue = None strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey if (strDataOutputKey in self.__dictXSDataOutput.keys()): oValue = self.__dictXSDataOutput[strDataOutputKey] return oValue def hasDataOutput(self, _strDataOutputKey=None): """ Returns True if the plugin has the specified Output Data """ strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey if (strDataOutputKey in self.__dictXSDataOutput.keys()): return True else: return False def delDataOutput(self, _strDataOutputKey=None): """ Deletes the data output for a particular key. If the key is not provided a default key is used. """ strDataOutputKey = _strDataOutputKey if (strDataOutputKey is None): strDataOutputKey = self.__strDefaultOutputDataKey if (strDataOutputKey in self.__dictXSDataOutput.keys()): self.__dictXSDataOutput[strDataOutputKey] = None else: strErrorMessage = self.getPluginName( ) + ".delDataOutput, no output data defined for key: " + _strDataOutputKey self.warning(strErrorMessage) self.addWarningMessage(strErrorMessage) # Property for dataOutput dataOutput = property(getDataOutput, setDataOutput, delDataOutput, "Property for dataOutput") def exportDataOutput(self, _edPlugin=None): """ Deprecated Exports the Plugin Output Data to slot """ self.DEBUG("EDPlugin.exportDataOutput") self.__edSlotExportDataOutput.call(self.__dictXSDataOutput) def connectExportDataOutput(self, _oMethod): """ Deprecated """ self.synchronizeOn() if (_oMethod != None): self.__edSlotExportDataOutput.connect(_oMethod) self.synchronizeOff() def generateExecutiveSummary(self, _edPlugin): """ This method, which should be implemented by sub-classes, generates an executive summary (user-related output summary). """ self.DEBUG("EDPlugin.generateExecutiveSummary") def addErrorMessage(self, _strErrorMessage): """ Adds an error message to the error messages list """ self.__listErrorMessages.append(_strErrorMessage) def getListOfErrorMessages(self): """ Returns the error messages list """ return self.__listErrorMessages def addWarningMessage(self, _strWarningMessage): """ Adds a warning message to the warning messages list """ self.DEBUG("EDPlugin.addWarningMessage : " + _strWarningMessage) self.__listWarningMessages.append(_strWarningMessage) def getListOfWarningMessages(self): """ Returns the warning messages list """ return self.__listWarningMessages def writeDataInput(self, _edObject=None): """ Writes the input data object(s) into a working dir xml file """ self.DEBUG("EDPlugin.writeDataInput") strBasename = os.path.join( self.getWorkingDirectory(), self.compactPluginName(self.getPluginName())) for strKey in self.__dictXSDataInput.keys(): if (strKey == self.__strDefaultInputDataKey): # "Old" style xsDataInput = self.__dictXSDataInput[ self.__strDefaultInputDataKey] self.strPathDataInput = strBasename + "_dataInput.xml" EDUtilsFile.writeFile(self.strPathDataInput, xsDataInput.marshal()) else: # We have a list of objects listXSDataInput = self.__dictXSDataInput[strKey] for iIndex, xsDataInput in enumerate(listXSDataInput): strPathDataInput = "%s_%s_%d_dataInput.xml" % ( strBasename, strKey, iIndex) EDUtilsFile.writeFile(strPathDataInput, xsDataInput.marshal()) def writeDataOutput(self, _edObject=None): """ Writes the output data object(s) into a working dir xml file """ self.DEBUG("EDPlugin.writeDataOutput") for strKey in self.__dictXSDataOutput.keys(): if (strKey == self.__strDefaultOutputDataKey): # "Old" style xsDataOutput = self.__dictXSDataOutput[ self.__strDefaultOutputDataKey] if (xsDataOutput is not None): self.strPathDataOutput = os.path.join( self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()) + "_dataOutput.xml") EDUtilsFile.writeFile(self.strPathDataOutput, xsDataOutput.marshal()) else: listXSDataOutput = self.__dictXSDataOutput[strKey] for iIndex, xsDataOutput in enumerate(listXSDataOutput): if (xsDataOutput is not None): strPathDataOutput = os.path.join( self.getWorkingDirectory(), self.compactPluginName(self.getPluginName()) + "_" + strKey + "_%d_dataOutput.xml" % iIndex) EDUtilsFile.writeFile(strPathDataOutput, xsDataOutput.marshal()) def getBaseName(self): """ Returns the plugin base name """ if (self.__strBaseName is None): self.__strBaseName = self.createBaseName() return self.__strBaseName def setBaseName(self, _strBaseName): """ Sets the plugin base name """ self.__strBaseName = self.compactPluginName(_strBaseName) self.setName(self.__strBaseName) # Create the directory baseDirectory/baseName which will be used as working directory strWorkingDirPath = os.path.join(self.getBaseDirectory(), self.__strBaseName) if not os.path.isdir(strWorkingDirPath): os.mkdir(strWorkingDirPath) self.setWorkingDirectory(strWorkingDirPath) def createBaseName(self): """ Generates the plugin base name: (<prefix>-<object ID>) """ # First try to use global instance ID from EDObject strBaseName = "%s-%08d" % (self.compactPluginName( self.getPluginName()), self.getId()) strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName) # Try to create the directory... try: os.mkdir(strBaseDir) except Exception as strErrorDetail: self.error( "EDPlugin.createBaseName: Could not create base directory %s because of %s" % (strBaseDir, strErrorDetail)) self.warning( "EDPlugin.createBaseName: Trying to create alternative base directory..." ) self.writeErrorTrace() strTempDir = tempfile.mkdtemp(prefix=strBaseName, dir=self.getBaseDirectory()) os.chmod( strTempDir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) strBaseName = os.path.split(strTempDir)[1] strBaseDir = os.path.join(self.getBaseDirectory(), strBaseName) self.warning( "EDPlugin.createBaseName: Alternative base directory created: %s" % strBaseDir) self.DEBUG("EDPlugin.createBaseName : Directory created = " + strBaseDir) return strBaseName def compactPluginName(self, _pluginName): """ The prefix is constructed from the plugin name with the following renaming: EDPlugin -> "" EDPluginExec -> "" EDPluginControl -> "Control" """ strCompactName = _pluginName if (strCompactName.startswith("EDPluginExec")): strCompactName = strCompactName[12:] elif (strCompactName.startswith("EDPluginControl")): strCompactName = strCompactName[8:] elif (strCompactName.startswith("EDPlugin")): strCompactName = strCompactName[8:] return strCompactName def setBaseDirectory(self, _strBaseDirectory): """ Sets the plugin base directory """ self.DEBUG("EDPlugin.setBaseDirectory : " + _strBaseDirectory) if (not os.path.isdir(_strBaseDirectory)): self.DEBUG( "EDPlugin.setBaseDirectory: base directory %s does dot yet exist! Creating it." % _strBaseDirectory) os.mkdir(_strBaseDirectory) self.__strBaseDirectory = _strBaseDirectory def getBaseDirectory(self): """ Returns the plugin base directory """ self.DEBUG("EDPlugin.getBaseDirectory : %s" % self.__strBaseDirectory) if (self.__strBaseDirectory is None): self.__strBaseDirectory = os.getcwd() return self.__strBaseDirectory def setWorkingDirectory(self, _strWorkingDirectory): """ Sets the plugin working directory """ self.DEBUG("EDPlugin.setWorkingDirectory : " + _strWorkingDirectory) self.__strWorkingDirectory = _strWorkingDirectory if not os.path.isdir(_strWorkingDirectory): self.DEBUG( "EDPlugin.setWorkingDirectory, creating working directory %s." % _strWorkingDirectory) os.mkdir(self.__strWorkingDirectory) def getWorkingDirectory(self): """ Returns the plugin base directory """ self.DEBUG("EDPlugin.getWorkingDirectory : %s" % self.__strWorkingDirectory) returnValue = None if (self.__strWorkingDirectory is not None): returnValue = self.__strWorkingDirectory return returnValue def checkMandatoryParameters(self, _xsData, _strParamName): """ Checks that a mandatory parameter exists in the data If not, an error message is added in the list and the plugin fails """ if _xsData is None or (hasattr(_xsData, '__len__') and len(_xsData) == 0): strErrorMessage = "%s: input parameter is missing: %s" % ( self.getPluginName(), _strParamName) self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) raise RuntimeError(strErrorMessage) def checkImportantParameters(self, _xsData, _strParamName): """ Checks that a specific parameter exists in the data If not, a warning message is added in the list """ if (_xsData == None): strWarningMessage = "%s: input parameter is missing: %s" % ( self.getPluginName(), _strParamName) self.warning(strWarningMessage) self.addWarningMessage(strWarningMessage) def addExecutiveSummaryLine(self, _strExecutiveSummaryLine): """ Add a line to the executive summary string. """ self.DEBUG("EDPlugin.addExecutiveSummaryLine : %r" % _strExecutiveSummaryLine) strExecutiveSummaryLine = _strExecutiveSummaryLine if (not strExecutiveSummaryLine == ""): if (strExecutiveSummaryLine[-1] == "\n"): strExecutiveSummaryLine = strExecutiveSummaryLine[:-1] self.__listExecutiveSummaryLines.append(strExecutiveSummaryLine) def addExecutiveSummarySeparator(self, _strSeparator=None): """ Adds a separator to split the executive summary into different parts Default is a dotted line """ strSeparator = _strSeparator if (strSeparator is None): strSeparator = self.__strExecutiveSummarySeparator # Check that the last line doesn't already contain a separator if self.__listExecutiveSummaryLines != []: if self.__listExecutiveSummaryLines[-1] != strSeparator: self.addExecutiveSummaryLine(strSeparator) else: self.addExecutiveSummaryLine(strSeparator) def getListExecutiveSummaryLines(self): """ Returns the executive summary (list of text lines) """ return self.__listExecutiveSummaryLines def verboseScreenExecutiveSummary(self): """ Prints the executive summary on the screen """ for line in self.getListExecutiveSummaryLines(): self.screen(line) def verboseDebug(self, _strMessage): self.DEBUG(self.getPluginName() + " : " + _strMessage) def getPluginName(self): return self.getClassName() def getListOfDataInputKeys(self): return self.__dictXSDataInput.keys() def getListOfDataOutputKeys(self): return self.__dictXSDataOutput.keys() def getDefaultInputDataKey(self): return self.__strDefaultInputDataKey def getDefaultOutputDataKey(self): return self.__strDefaultOutputDataKey def getExecutiveSummarySeparator(self): return self.__strExecutiveSummarySeparator def isRequiredToHaveConfiguration(self): """ If the return value from this method is true, the plugin is required to have a configuration in order to be executed in a plugin execution test case. @return: RequiredToHaveConfiguration @rtype: boolean """ return self.__isRequiredToHaveConfiguration def setRequiredToHaveConfiguration(self, _bValue=True): """ Sets or unsets the plugin to be required to have a configuration for execution in a plugin execution test case. plugin execution test case. @param _bValue: RequiredToHaveConfiguration @type: boolean """ self.__isRequiredToHaveConfiguration = _bValue def setWriteXMLInputOutput(self, _bValue=True): """ Sets or unsets the plugin to write XML input and output files. @param _bValue: WriteDataXMLInputOutput @type: boolean """ self.__bWriteDataXMLInputOutput = _bValue def setWriteXMLInput(self, _bValue=True): """ Sets or unsets the plugin to write XML input files. @param _bValue: WriteDataXMLInput @type: boolean """ self.__bWriteDataXMLInput = _bValue def setWriteXMLOutput(self, _bValue=True): """ Sets or unsets the plugin to write XML output files. @param _bValue: WriteDataXMLOutput @type: boolean """ self.__bWriteDataXMLOutput = _bValue def setUseWarningInsteadOfError(self, _bValue=True): """ Sets or unsets the plugin to use warning messages also for error messages. @param _bValue: UseWarningInsteadOfError @type: boolean """ self.__bUseWarningInsteadOfError = _bValue def error(self, _strErrorMessage): """ Overloaded from EDLogging. If self.__bUseWarningInsteadOfError is True a warning message is issued instead of an error message. """ if self.__bUseWarningInsteadOfError: self.warning(_strErrorMessage) else: EDAction.error(self, _strErrorMessage) def ERROR(self, _strErrorMessage): """ Uses the overloaded self.error method above. """ self.error(_strErrorMessage)
class EDJob(EDLogging): """ Create a module called EDJob * Most of what was done up to 09-2010 in EDParallelExecute should be done here * Each instance will be a job * Constructor takes a plugin name * Each instance will have taking an "setDataInput" method getting an XMLin (as string) * Each instance will gave a "getDataOutput" method with optional join * there could be a "join" method, waiting for the job to finish * Each instance will have a "execute" method and returning a JobId * Each instance will have a "setCallBack" method that stores the name of the external callback * provide status of a job * keep track of all plugin status * leave the time to plugin to initialize * static class retrieve job-instance, status, small-log ... * prevent multiple run of a single job ? * does not manage workload of the computer, should be managed at the ExecPlugin level Used for the tango binding, EDParallelExecute ... == class variables == dictPluginStatus[pluginName] = ["uninitialized"|"running"|"executed"| "failed"] dictJobs [JobId] = EDJob.Instance == static methods == getJob(JobId) """ PLUGIN_STATE_UNITIALIZED = "uninitialized" PLUGIN_STATE_RUNNING = "running" PLUGIN_STATE_SUCCESS = "success" PLUGIN_STATE_FAILURE = "failure" __edFactoryPlugin = EDFactoryPlugin() __dictJobs = {} __semaphore = Semaphore() __fStartTime = time.time() def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDLogging.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__pathXSDInput = None self.__pathXSDOutput = None self.__bXmlInputSet = False self.__status = None self.__name = None self.__runtime = None self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin( self.__strPluginName) if self.__edPlugin is None: raise RuntimeError("Unable to create plugin %s" % self.__strPluginName) self.__jobId = "%s-%08i" % (self.__strPluginName, self.__edPlugin.getId()) with self.__class__.__semaphore: self.__class__.__dictJobs[self.__jobId] = self if (self.__edPlugin is None): self.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the job (plugin) input data. @param: _oDataInput: could be either an String XML or an XSData object. @param _strDataInputKey: the key of an input data dictionnary The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ if _oDataInput in ["", None]: self.__bXmlInputSet = False return else: with self.locked(): if (self.__edPlugin is not None): self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey) self.__bXmlInputSet = True else: self.WARNING( "Setting DataInput for uninstanciated plugin %s." % self.__strPluginName) def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ if (self.__edPlugin is not None): return self.__edPlugin.getDataInput(_strDataInputKey).marshal() elif (self.__pathXSDInput is not None): return open(self.__pathXSDInput).read() else: self.WARNING("Getting DataInput for uninstanciated plugin %s." % self.__strPluginName) dataInput = property(getDataInput, setDataInput) def getDataOutput(self, _strDataOutputKey=None, _bWait=True): """ Returns the Plugin Output Data @param _bWait: shall we wait for the plugin to finish to retrieve output data: Yes by default. @type _bWait: boolean """ if _bWait: # Wait for plugin to finish befor returning data output self.synchronize() if (self.__edPlugin is not None): return self.__edPlugin.getDataOutput(_strDataOutputKey).marshal() elif self.__pathXSDOutput is not None: return open(self.__pathXSDOutput).read() else: self.WARNING( "Getting DataOutput for uninstanciated plugin or plugin has been garbage collected or output data were not saved. JobID was %s ." % self.__jobId) dataOutput = property(getDataOutput) def execute(self): """ Launch the EDNA plugin @return: JobId @rtype: string """ if not self.__bXmlInputSet: self.WARNING("Not executing job %s as input is empty" % self.__jobId) if (self.__edPlugin is not None): with self.locked(): self.__edPlugin.connectSUCCESS(self.successPluginExecution) self.__edPlugin.connectFAILURE(self.failurePluginExecution) self.__status = EDJob.PLUGIN_STATE_RUNNING self.__edPlugin.execute() return self.__jobId else: self.WARNING("Trying to run a plugin that does not exist: %s " % self.__strPluginName) def synchronize(self): """ Synchronize the execution of the job with the calling thread. """ with self.locked(): strStatus = self.__status if strStatus == EDJob.PLUGIN_STATE_RUNNING: self.__edPlugin.synchronize() elif strStatus == EDJob.PLUGIN_STATE_UNITIALIZED: self.WARNING("Unable to synchronize %s jobs" % strStatus) else: self.DEBUG("Unable to synchronize %s jobs" % strStatus) @classmethod def synchronizeAll(cls): """ Wait for all jobs to finish. """ EDVerbose.DEBUG("EDJob.synchronizeAll class method ") listJob = cls.__dictJobs.keys() for jobid in listJob: job = cls.__dictJobs[jobid] job.synchronize() if len(cls.__dictJobs) != len(listJob): EDVerbose.WARNING( "EDJob.synchronizeAll: New jobs have been launched while synchronizing" ) def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ with self.locked(): self.__status = EDJob.PLUGIN_STATE_SUCCESS self.screen("Plugin %s: success after %.3fs" % (self.__jobId, _edObject.getRunTime())) try: self.__edSlotSUCCESS.call(self.__jobId) except Exception: self.ERROR("Error in execution of Success call-back for %s" % self.__jobId) self.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: self.ERROR( "Error in execution of Common call-back (after success) for %s" % self.__jobId) self.writeErrorTrace() def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ with self.locked(): self.__status = EDJob.PLUGIN_STATE_FAILURE self.screen("Plugin %s: failure after %.3fs" % (self.__jobId, _edObject.getRunTime())) try: self.__edSlotFAILURE.call(self.__jobId) except Exception: self.ERROR("Error in execution of Failure call-back for %s" % self.__jobId) self.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: self.ERROR( "Error in execution of Common call-back (after failure) for %s" % self.__jobId) self.writeErrorTrace() def connectSUCCESS(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotSUCCESS.connect(_oMethod) def connectFAILURE(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotFAILURE.connect(_oMethod) def connectCallBack(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotCallBack.connect(_oMethod) def getJobId(self): """ @return: JobId i.e. EDPluginName-Number @rtype: string """ return self.__jobId jobId = property(getJobId, "EDJob.jobId: read-only property") getJobID = getJobId def getPluginName(self): """ @return: Name of the plugin @rtype: string """ return self.__strPluginName pluginName = property(getPluginName, "EDJob.pluginName: read-only property") def getPlugin(self): """ @return: the plugin (instance) @rtype: python object """ return self.__edPlugin plugin = property(getPlugin, "EDJob.plugin: read-only property") def getStatus(self): """ @return: status of the Job @rtype: string """ return self.__status status = property(getStatus, "EDJob.status: read-only property") def getName(self): return self.__name def setName(self, _strName): if self.__name is None: self.__name = _strName else: self.WARNING("EDJob.setName: One cannot rename a Job !!!") name = property(getName, setName, "EDJob.name: nickname of the job") def getMemSize(self): """ try to guess the size in memory of a job @return: expected size in memory """ if asizeof is not None: return asizeof.asizeof(self) @classmethod def getStatusFromID(cls, jobId): """ Retrieve the job (hence the plugin) status @param jobId: the Job identification number @type jobId: string @return: the EDJob status @rtype: string """ if jobId in cls.__dictJobs: strRet = cls.__dictJobs[jobId].getStatus() else: strRet = "Unable to retrieve such job: %s" % jobId EDVerbose.WARNING(strRet) return strRet getStatusFromId = getStatusFromID @classmethod def getJobFromID(cls, jobId): """ Retrieve the job (hence the plugin) @param jobId: the Job identification number @type jobId: string @return: the "EDJob instance", which contains the plugin (__edPlugin) and the status @rtype: a Python object. """ if jobId in cls.__dictJobs: return cls.__dictJobs[jobId] else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) getJobFromId = getJobFromID @classmethod def getMemoryFootprint(cls): if asizeof is not None: return asizeof.asizesof(cls.__dictJobs) def cleanJob(self, forceGC=True): """ Frees the memory associated with the top level plugin @param forceGC: Force garbage collection after clean-up @type forceGC: boolean """ self.synchronize() with self.locked(): if self.__edPlugin is not None: self.__pathXSDOutput = self.__edPlugin.strPathDataOutput self.__pathXSDInput = self.__edPlugin.strPathDataInput self.__runtime = self.__edPlugin.getRunTime() self.__edPlugin = None if forceGC: gc.collect() @classmethod def cleanJobfromId(cls, jobId, forceGC=True): """ Frees the memory associated with the top level plugin @param jobId: the Job identification number @type jobId: string @param forceGC: Force garbage collection after clean-up @type forceGC: boolean """ if jobId in cls.__dictJobs: job = cls.__dictJobs[jobId] job.cleanJob(forceGC) strRet = "Job %s cleaned" % jobId else: strRet = "Unable to retrieve such EDJob: %s" % jobId EDVerbose.WARNING(strRet) return strRet cleanJobfromID = cleanJobfromId @classmethod def getDataOutputFromId(cls, jobId): """ Returns the Plugin Output Data @param jobId: job idenfier @type jobId: string @return: edJob.DataOutput XML string """ output = None job = cls.getJobFromId(jobId) if job is not None: output = job.getDataOutput() return output or "" getDataOutputFromID = getDataOutputFromId @classmethod def getDataInputFromId(cls, jobId): """ Returns the Plugin Input Data @param jobId: job idenfier @type jobId: string @return: edJob.DataInput XML string """ output = None job = cls.getJobFromId(jobId) if job is not None: output = job.getDataInput() return output or "" getDataInputFromID = getDataInputFromId @classmethod def countRunning(cls): """ return the number of jobs still running. """ running = 0 for jobid in list(cls.__dictJobs.keys()): # python3 ! running += ( cls.__dictJobs[jobid].__status == cls.PLUGIN_STATE_RUNNING) return running @classmethod def stats(cls): """ Retrieve some statistics and print them """ lstStrOut = [] output = [] fExecTime = time.time() - cls.__fStartTime keys = cls.__dictJobs.keys() keys.sort() for num, key in enumerate(keys): job = cls.__dictJobs[key] if job.getPlugin() is None: runtime = job.__runtime else: runtime = job.getPlugin().getRunTime() output.append( [num, key, job.getStatus(), runtime, job.getMemSize()]) output.sort() iNbJob = max(1, len(keys)) lstStrOut.append("_" * 110) lstStrOut.append( "%s\t|\t%s\t\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("nr", "EDPluginName-Id", "status", "runtime", "memory")) lstStrOut.append("_" * 110) fWall = 0.0 fSumProd = 0.0 fSumX = 0.0 fSumX2 = 0.0 for oneJob in output: fWall += oneJob[3] fSumX += oneJob[0] fSumX2 += oneJob[0] * oneJob[0] fSumProd += oneJob[0] * oneJob[3] lstStrOut.append("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob)) lstStrOut.append("_" * 110) lstStrOut.append( "Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime)) lstStrOut.append( "Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob)) if len(keys) > 1: fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX) fOrd = (fWall - fSlope * fSumX) / iNbJob else: fSlope = 0.0 fOrd = fWall lstStrOut.append( "Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope)) strOutput = os.linesep.join(lstStrOut) EDVerbose.screen(strOutput) return strOutput
def __init__(self): EDLogging.__init__(self) Thread.__init__(self) self.__edSlotPreProcess = EDSlot() self.__edSlotProcess = EDSlot() self.__edSlotPostProcess = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__edSlotFinallyProcess = EDSlot() self.__bIsFailure = False self.__bIsTimeOut = False self.__fTimeOutInSeconds = None self.__fDefaultTimeOutInSeconds = 600.0 self.__bIsAbort = False # Reference to the object which calls execute or executeSynchronous self.__edObject = None self.__lExtraTime = [ ] # list of extra allowed time for execution (in second) self.__bLogTiming = False