def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDObject.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__bXmlInputSet = False self.__status = None # self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName) self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName) EDJob.__semaphore.acquire() # Create the JobID if not self.__strPluginName in EDJob.__dictPluginLastId: EDJob.__dictPluginLastId[_strPluginName] = 0 else: EDJob.__dictPluginLastId[_strPluginName] += 1 self.__jobId = "%s-%i" % (self.__strPluginName, EDJob.__dictPluginLastId[_strPluginName]) EDJob.__dictJobs[self.__jobId] = self EDJob.__semaphore.release() if (self.__edPlugin is None): EDVerbose.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def testRegisterPluginWrapper(self): edServer = EDServerXMLRCP.getInstance() strPluginName = "EDPluginTestPluginFactory" edPlugin = EDPluginWrapperForJobScheduler(strPluginName) edServer.registerPlugin(edPlugin) strId = str(edPlugin.getId()) edPluginRegistered = edServer.getRegisteredPlugin(strId) if not edPluginRegistered: EDAssert.equal(not None, edPluginRegistered, "Registered plugin") edServer.unRegisterPlugin(edPlugin) edPluginRegistered = edServer.getRegisteredPlugin(strId) EDAssert.equal(None, edPluginRegistered, "Plugin unregistered")
def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDObject.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__bXmlInputSet = False self.__status = None # self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName) self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName) EDJob.__semaphore.acquire() # Create the JobID if not self.__strPluginName in EDJob.__dictPluginLastId: EDJob.__dictPluginLastId[_strPluginName] = 0 else: EDJob.__dictPluginLastId[_strPluginName] += 1 self.__jobId = "%s-%i" % (self.__strPluginName, EDJob.__dictPluginLastId[_strPluginName]) EDJob.__dictJobs[self.__jobId] = self EDJob.__semaphore.release() if (self.__edPlugin is None): EDVerbose.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def testCallbackSUCCESS(self): edServer = EDServerXMLRCP.getInstance() strPluginName = "EDPluginTestPluginFactory" edPlugin = EDPluginWrapperForJobScheduler(strPluginName) edPlugin.connectSUCCESS(self.callBackSUCCESS) edServer.registerPlugin(edPlugin) edPlugin.executeSynchronous() edServer.shutdown()
class EDJob(EDObject): PLUGIN_STATE_UNITIALIZED = "uninitialized" PLUGIN_STATE_RUNNING = "running" PLUGIN_STATE_SUCCESS = "success" PLUGIN_STATE_FAILURE = "failure" __edFactoryPlugin = EDFactoryPlugin() __dictJobs = {} __semaphore = threading.Semaphore() __dictPluginLastId = {} __fStartTime = time.time() def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDObject.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__bXmlInputSet = False self.__status = None # self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName) self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName) EDJob.__semaphore.acquire() # Create the JobID if not self.__strPluginName in EDJob.__dictPluginLastId: EDJob.__dictPluginLastId[_strPluginName] = 0 else: EDJob.__dictPluginLastId[_strPluginName] += 1 self.__jobId = "%s-%i" % (self.__strPluginName, EDJob.__dictPluginLastId[_strPluginName]) EDJob.__dictJobs[self.__jobId] = self EDJob.__semaphore.release() if (self.__edPlugin is None): EDVerbose.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the job (plugin) input data. @param: _oDataInput: could be either an String XML or an XSData object. @param _strDataInputKey: the key of an input data dictionnary The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ if _oDataInput in ["", None]: self.__bXmlInputSet = False return self.synchronizeOn() if (self.__edPlugin is not None): self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey) self.__bXmlInputSet = True else: EDVerbose.WARNING("Setting DataInput for uninstanciated plugin %s." % self.__strPluginName) self.synchronizeOff() def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ if (self.__edPlugin is not None): return self.__edPlugin.getDataInput(_strDataInputKey) else: EDVerbose.WARNING("Getting DataInput for uninstanciated plugin %s." % self.__strPluginName) def getDataOutput(self, _strDataOutputKey=None): """ Returns the Plugin Output Data """ if (self.__edPlugin is not None): return self.__edPlugin.getDataOutput(_strDataOutputKey) else: EDVerbose.WARNING("Getting DataOutput for uninstanciated plugin %s." % self.__strPluginName) def execute(self): """ Launch the EDNA plugin @return: JobId @rtype: string """ returnId = None if not self.__bXmlInputSet: EDVerbose.WARNING("Not executing job %s as input is empty" % self.__jobId) if (self.__edPlugin is not None): self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_RUNNING self.__edPlugin.connectSUCCESS(self.successPluginExecution) self.__edPlugin.connectFAILURE(self.failurePluginExecution) self.__edPlugin.execute() returnId = self.__jobId self.synchronizeOff() else: EDVerbose.WARNING("Trying to run a plugin that does not exist: %s " % self.__strPluginName) return returnId def synchronize(self): """ Synchronize the execution of the job with the calling thread. """ self.__edPlugin.synchronize() def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_SUCCESS EDVerbose.screen("Plugin %s execution ended with success" % self.__jobId) self.synchronizeOff() try: self.__edSlotSUCCESS.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Success call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Common call-back (after success) for %s" % self.__jobId) EDVerbose.writeErrorTrace() def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_FAILURE EDVerbose.screen("Plugin %s execution ended with failure" % self.__jobId) self.synchronizeOff() try: self.__edSlotFAILURE.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Failure call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Common call-back (after failure) for %s" % self.__jobId) EDVerbose.writeErrorTrace() def connectSUCCESS(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotSUCCESS.connect(_oMethod) self.synchronizeOff() def connectFAILURE(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotFAILURE.connect(_oMethod) self.synchronizeOff() def connectCallBack(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotCallBack.connect(_oMethod) self.synchronizeOff() def getJobId(self): """ @return: JobId i.e. EDPluginName-Number @rtype: string """ return self.__jobId def getPluginName(self): """ @return: Name of the plugin @rtype: string """ return self.__strPluginName def getPlugin(self): """ @return: the plugin (instance) @rtype: python object """ return self.__edPlugin def getStatus(self): """ @return: status of the Job @rtype: string """ return self.__status def getMemSize(self): """ try to guess the size in memory of a job @return: expected size in memory """ if asizeof is not None: return asizeof.asizeof(self) @staticmethod def getStatusFromID(jobId): """ Retrieve the job (hence the plugin) status @param jobId: the Job identification number @type jobId: string @return: the EDJob status @rtype: string """ if jobId in EDJob.__dictJobs: return EDJob.__dictJobs[jobId].getStatus() else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) @staticmethod def getJobFromID(jobId): """ Retrieve the job (hence the plugin) @param jobId: the Job identification number @type jobId: string @return: the "EDJob instance", which contains the plugin (__edPlugin) and the status @rtype: a Python object. """ if jobId in EDJob.__dictJobs: return EDJob.__dictJobs[jobId] else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) @staticmethod def getMemoryFootprint(): if asizeof is not None: return asizeof.asizesof(EDJob.__dictJobs) @staticmethod def stats(): """ retrieve some statistics """ output = [] fExecTime = time.time() - EDJob.__fStartTime keys = EDJob.__dictJobs.keys() for key in keys : num = int(key.split("-", 1)[1]) job = EDJob.__dictJobs[key] output.append([num, job.getPluginName(), job.getStatus(), job.getPlugin().getRunTime(), job.getMemSize()]) output.sort() iNbJob = max(1, len(output)) EDVerbose.screen("%s\t|\t%s\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("id", "EDPluginName", "status", "runtime", "memory")) fWall = 0.0 fSumProd = 0.0 fSumX = 0.0 fSumX2 = 0.0 for oneJob in output: fWall += oneJob[3] fSumX += oneJob[0] fSumX2 += oneJob[0] * oneJob[0] fSumProd += oneJob[0] * oneJob[3] EDVerbose.screen("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob)) EDVerbose.screen("_" * 90) EDVerbose.screen("Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime)) EDVerbose.screen("Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob)) fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX) fOrd = (fWall - fSlope * fSumX) / iNbJob EDVerbose.screen("Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope))
class EDJob(EDObject): PLUGIN_STATE_UNITIALIZED = "uninitialized" PLUGIN_STATE_RUNNING = "running" PLUGIN_STATE_SUCCESS = "success" PLUGIN_STATE_FAILURE = "failure" __edFactoryPlugin = EDFactoryPlugin() __dictJobs = {} __semaphore = threading.Semaphore() __dictPluginLastId = {} __fStartTime = time.time() def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDObject.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__bXmlInputSet = False self.__status = None # self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName) self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName) EDJob.__semaphore.acquire() # Create the JobID if not self.__strPluginName in EDJob.__dictPluginLastId: EDJob.__dictPluginLastId[_strPluginName] = 0 else: EDJob.__dictPluginLastId[_strPluginName] += 1 self.__jobId = "%s-%i" % (self.__strPluginName, EDJob.__dictPluginLastId[_strPluginName]) EDJob.__dictJobs[self.__jobId] = self EDJob.__semaphore.release() if (self.__edPlugin is None): EDVerbose.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the job (plugin) input data. @param: _oDataInput: could be either an String XML or an XSData object. @param _strDataInputKey: the key of an input data dictionnary The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ if _oDataInput in ["", None]: self.__bXmlInputSet = False return self.synchronizeOn() if (self.__edPlugin is not None): self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey) self.__bXmlInputSet = True else: EDVerbose.WARNING( "Setting DataInput for uninstanciated plugin %s." % self.__strPluginName) self.synchronizeOff() def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ if (self.__edPlugin is not None): return self.__edPlugin.getDataInput(_strDataInputKey) else: EDVerbose.WARNING( "Getting DataInput for uninstanciated plugin %s." % self.__strPluginName) def getDataOutput(self, _strDataOutputKey=None): """ Returns the Plugin Output Data """ if (self.__edPlugin is not None): return self.__edPlugin.getDataOutput(_strDataOutputKey) else: EDVerbose.WARNING( "Getting DataOutput for uninstanciated plugin %s." % self.__strPluginName) def execute(self): """ Launch the EDNA plugin @return: JobId @rtype: string """ returnId = None if not self.__bXmlInputSet: EDVerbose.WARNING("Not executing job %s as input is empty" % self.__jobId) if (self.__edPlugin is not None): self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_RUNNING self.__edPlugin.connectSUCCESS(self.successPluginExecution) self.__edPlugin.connectFAILURE(self.failurePluginExecution) self.__edPlugin.execute() returnId = self.__jobId self.synchronizeOff() else: EDVerbose.WARNING( "Trying to run a plugin that does not exist: %s " % self.__strPluginName) return returnId def synchronize(self): """ Synchronize the execution of the job with the calling thread. """ self.__edPlugin.synchronize() def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_SUCCESS EDVerbose.screen("Plugin %s execution ended with success" % self.__jobId) self.synchronizeOff() try: self.__edSlotSUCCESS.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Success call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR( "Error in execution of Common call-back (after success) for %s" % self.__jobId) EDVerbose.writeErrorTrace() def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_FAILURE EDVerbose.screen("Plugin %s execution ended with failure" % self.__jobId) self.synchronizeOff() try: self.__edSlotFAILURE.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Failure call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR( "Error in execution of Common call-back (after failure) for %s" % self.__jobId) EDVerbose.writeErrorTrace() def connectSUCCESS(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotSUCCESS.connect(_oMethod) self.synchronizeOff() def connectFAILURE(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotFAILURE.connect(_oMethod) self.synchronizeOff() def connectCallBack(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotCallBack.connect(_oMethod) self.synchronizeOff() def getJobId(self): """ @return: JobId i.e. EDPluginName-Number @rtype: string """ return self.__jobId def getPluginName(self): """ @return: Name of the plugin @rtype: string """ return self.__strPluginName def getPlugin(self): """ @return: the plugin (instance) @rtype: python object """ return self.__edPlugin def getStatus(self): """ @return: status of the Job @rtype: string """ return self.__status def getMemSize(self): """ try to guess the size in memory of a job @return: expected size in memory """ if asizeof is not None: return asizeof.asizeof(self) @staticmethod def getStatusFromID(jobId): """ Retrieve the job (hence the plugin) status @param jobId: the Job identification number @type jobId: string @return: the EDJob status @rtype: string """ if jobId in EDJob.__dictJobs: return EDJob.__dictJobs[jobId].getStatus() else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) @staticmethod def getJobFromID(jobId): """ Retrieve the job (hence the plugin) @param jobId: the Job identification number @type jobId: string @return: the "EDJob instance", which contains the plugin (__edPlugin) and the status @rtype: a Python object. """ if jobId in EDJob.__dictJobs: return EDJob.__dictJobs[jobId] else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) @staticmethod def getMemoryFootprint(): if asizeof is not None: return asizeof.asizesof(EDJob.__dictJobs) @staticmethod def stats(): """ retrieve some statistics """ output = [] fExecTime = time.time() - EDJob.__fStartTime keys = EDJob.__dictJobs.keys() for key in keys: num = int(key.split("-", 1)[1]) job = EDJob.__dictJobs[key] output.append([ num, job.getPluginName(), job.getStatus(), job.getPlugin().getRunTime(), job.getMemSize() ]) output.sort() iNbJob = max(1, len(output)) EDVerbose.screen("%s\t|\t%s\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("id", "EDPluginName", "status", "runtime", "memory")) fWall = 0.0 fSumProd = 0.0 fSumX = 0.0 fSumX2 = 0.0 for oneJob in output: fWall += oneJob[3] fSumX += oneJob[0] fSumX2 += oneJob[0] * oneJob[0] fSumProd += oneJob[0] * oneJob[3] EDVerbose.screen("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob)) EDVerbose.screen("_" * 90) EDVerbose.screen( "Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime)) EDVerbose.screen( "Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob)) fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX) fOrd = (fWall - fSlope * fSumX) / iNbJob EDVerbose.screen( "Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope))