def appendPath(self, _strPluginTestLocation): """ For the tests, both the plugin directory and its corresponding "src" directory must be on the python path (see the class documentation for EDFactoryPluginTest). This method appends the plugin "src" directory to the system path, if it's not already present. @param _strModuleLocation: Path to the module location @type _strModuleLocation: python string """ EDFactoryPlugin.appendPath(self, _strPluginTestLocation) strSrcDirectory = EDUtilsPath.appendListOfPaths( _strPluginTestLocation, ["..", "..", "..", "..", "src"]) if os.path.exists(strSrcDirectory): if (not strSrcDirectory in sys.path): sys.path.append(strSrcDirectory) strSrcDirectory = EDUtilsPath.appendListOfPaths( _strPluginTestLocation, ["..", "..", "src"]) if os.path.exists(strSrcDirectory): if (not strSrcDirectory in sys.path): sys.path.append(strSrcDirectory) strPluginDirectory = EDUtilsPath.appendListOfPaths( _strPluginTestLocation, ["..", "..", "plugins"]) if os.path.exists(strPluginDirectory): if (not strPluginDirectory in sys.path): sys.path.append(strPluginDirectory)
def appendPath(self, _strPluginTestLocation): """ For the tests, both the plugin directory and its corresponding "src" directory must be on the python path (see the class documentation for EDFactoryPluginTest). This method appends the plugin "src" directory to the system path, if it's not already present. @param _strModuleLocation: Path to the module location @type _strModuleLocation: python string """ EDFactoryPlugin.appendPath(self, _strPluginTestLocation) strSrcDirectory = EDUtilsPath.appendListOfPaths(_strPluginTestLocation, [ "..", "..", "..", "..", "src" ]) if os.path.exists(strSrcDirectory): if (not strSrcDirectory in sys.path): sys.path.append(strSrcDirectory) strSrcDirectory = EDUtilsPath.appendListOfPaths(_strPluginTestLocation, [ "..", "..", "src" ]) if os.path.exists(strSrcDirectory): if (not strSrcDirectory in sys.path): sys.path.append(strSrcDirectory) strPluginDirectory = EDUtilsPath.appendListOfPaths(_strPluginTestLocation, [ "..", "..", "plugins" ]) if os.path.exists(strPluginDirectory): if (not strPluginDirectory in sys.path): sys.path.append(strPluginDirectory)
def testGetProjectRootDirectory(self): edFactoryPlugin = EDFactoryPlugin() strEdnaHome = EDUtilsPath.getEdnaHome() strTestProjectRootDirectoryReference = EDUtilsPath.appendListOfPaths(strEdnaHome, [ "kernel", "tests", "data", "EDFactoryPlugin", "testProject" ]) strTestProjectRootDirectory1 = edFactoryPlugin.getProjectRootDirectory("EDPluginTestPluginFactory") EDAssert.equal(strTestProjectRootDirectoryReference, strTestProjectRootDirectory1) strTestProjectRootDirectory2 = edFactoryPlugin.getProjectRootDirectory("XSDataTestProject") EDAssert.equal(strTestProjectRootDirectoryReference, strTestProjectRootDirectory2) strTestProjectRootDirectory3 = edFactoryPlugin.getProjectRootDirectory("PluginThatNotExists") EDAssert.equal(None, strTestProjectRootDirectory3)
def createPlugin(self): """ Creates a plugin instance """ edPlugin = None exceptionObject = None try: edFactoryPlugin = EDFactoryPlugin() edPlugin = edFactoryPlugin.loadPlugin(self.getPluginName()) except ImportError, exceptionObject: strWarningMessage = "Could not create the plugin: %s, reason: %s" % (self.getPluginName(), exceptionObject) EDVerbose.WARNING(strWarningMessage)
def testGetPathToProjectConfigurationFile(self): EDUtilsPath.setEdnaSite("TestSite") edFactoryPlugin = EDFactoryPlugin() strPathToConfigurationFile1 = edFactoryPlugin.getPathToProjectConfigurationFile("EDPluginTestPluginFactory") strPathToConfigurationFileReference1 = EDUtilsPath.appendListOfPaths(EDUtilsPath.getEdnaHome(), [ "edna-kernel", "tests", "data", "EDFactoryPlugin", \ "testProject", "conf", "XSConfiguration_TestSite.xml" ]) EDAssert.equal(strPathToConfigurationFileReference1, strPathToConfigurationFile1) EDUtilsPath.setEdnaSite("NonexistingTestSite") strPathToConfigurationFile2 = edFactoryPlugin.getPathToProjectConfigurationFile("EDPluginTestPluginFactory") strPathToConfigurationFileReference2 = None EDAssert.equal(strPathToConfigurationFileReference2, strPathToConfigurationFile2) # Since this is a static variable we need to reset it to None in order not to break any other tests... EDUtilsPath.setEdnaSite(None)
def createPlugin(self): """ Creates a plugin instance """ edPlugin = None exceptionObject = None try: edFactoryPlugin = EDFactoryPlugin() edPlugin = edFactoryPlugin.loadPlugin(self.getPluginName()) except ImportError(exceptionObject): strWarningMessage = "Could not create the plugin: %s, reason: %s" % (self.getPluginName(), exceptionObject) EDVerbose.WARNING(strWarningMessage) if edPlugin is None: if exceptionObject is None: EDVerbose.error("EDTestCasePlugin.createPlugin: Could not create plugin: " + self.getPluginName()) return edPlugin
def testGetProjectRootDirectory(self): edFactoryPlugin = EDFactoryPlugin() strEdnaHome = EDUtilsPath.getEdnaHome() strTestProjectRootDirectoryReference = EDUtilsPath.appendListOfPaths( strEdnaHome, ["kernel", "tests", "data", "EDFactoryPlugin", "testProject"]) strTestProjectRootDirectory1 = edFactoryPlugin.getProjectRootDirectory( "EDPluginTestPluginFactory") EDAssert.equal(strTestProjectRootDirectoryReference, strTestProjectRootDirectory1) strTestProjectRootDirectory2 = edFactoryPlugin.getProjectRootDirectory( "XSDataTestProject") EDAssert.equal(strTestProjectRootDirectoryReference, strTestProjectRootDirectory2) strTestProjectRootDirectory3 = edFactoryPlugin.getProjectRootDirectory( "PluginThatNotExists") EDAssert.equal(None, strTestProjectRootDirectory3)
def createPlugin(self): """ Creates a plugin instance """ edPlugin = None exceptionObject = None try: edFactoryPlugin = EDFactoryPlugin() edPlugin = edFactoryPlugin.loadPlugin(self.getPluginName()) except ImportError(exceptionObject): strWarningMessage = "Could not create the plugin: %s, reason: %s" % ( self.getPluginName(), exceptionObject) EDVerbose.WARNING(strWarningMessage) if edPlugin is None: if exceptionObject is None: EDVerbose.error( "EDTestCasePlugin.createPlugin: Could not create plugin: " + self.getPluginName()) return edPlugin
class EDJob(EDObject): PLUGIN_STATE_UNITIALIZED = "uninitialized" PLUGIN_STATE_RUNNING = "running" PLUGIN_STATE_SUCCESS = "success" PLUGIN_STATE_FAILURE = "failure" __edFactoryPlugin = EDFactoryPlugin() __dictJobs = {} __semaphore = threading.Semaphore() __dictPluginLastId = {} __fStartTime = time.time() def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDObject.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__bXmlInputSet = False self.__status = None # self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName) self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName) EDJob.__semaphore.acquire() # Create the JobID if not self.__strPluginName in EDJob.__dictPluginLastId: EDJob.__dictPluginLastId[_strPluginName] = 0 else: EDJob.__dictPluginLastId[_strPluginName] += 1 self.__jobId = "%s-%i" % (self.__strPluginName, EDJob.__dictPluginLastId[_strPluginName]) EDJob.__dictJobs[self.__jobId] = self EDJob.__semaphore.release() if (self.__edPlugin is None): EDVerbose.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the job (plugin) input data. @param: _oDataInput: could be either an String XML or an XSData object. @param _strDataInputKey: the key of an input data dictionnary The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ if _oDataInput in ["", None]: self.__bXmlInputSet = False return self.synchronizeOn() if (self.__edPlugin is not None): self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey) self.__bXmlInputSet = True else: EDVerbose.WARNING( "Setting DataInput for uninstanciated plugin %s." % self.__strPluginName) self.synchronizeOff() def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ if (self.__edPlugin is not None): return self.__edPlugin.getDataInput(_strDataInputKey) else: EDVerbose.WARNING( "Getting DataInput for uninstanciated plugin %s." % self.__strPluginName) def getDataOutput(self, _strDataOutputKey=None): """ Returns the Plugin Output Data """ if (self.__edPlugin is not None): return self.__edPlugin.getDataOutput(_strDataOutputKey) else: EDVerbose.WARNING( "Getting DataOutput for uninstanciated plugin %s." % self.__strPluginName) def execute(self): """ Launch the EDNA plugin @return: JobId @rtype: string """ returnId = None if not self.__bXmlInputSet: EDVerbose.WARNING("Not executing job %s as input is empty" % self.__jobId) if (self.__edPlugin is not None): self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_RUNNING self.__edPlugin.connectSUCCESS(self.successPluginExecution) self.__edPlugin.connectFAILURE(self.failurePluginExecution) self.__edPlugin.execute() returnId = self.__jobId self.synchronizeOff() else: EDVerbose.WARNING( "Trying to run a plugin that does not exist: %s " % self.__strPluginName) return returnId def synchronize(self): """ Synchronize the execution of the job with the calling thread. """ self.__edPlugin.synchronize() def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_SUCCESS EDVerbose.screen("Plugin %s execution ended with success" % self.__jobId) self.synchronizeOff() try: self.__edSlotSUCCESS.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Success call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR( "Error in execution of Common call-back (after success) for %s" % self.__jobId) EDVerbose.writeErrorTrace() def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_FAILURE EDVerbose.screen("Plugin %s execution ended with failure" % self.__jobId) self.synchronizeOff() try: self.__edSlotFAILURE.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Failure call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR( "Error in execution of Common call-back (after failure) for %s" % self.__jobId) EDVerbose.writeErrorTrace() def connectSUCCESS(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotSUCCESS.connect(_oMethod) self.synchronizeOff() def connectFAILURE(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotFAILURE.connect(_oMethod) self.synchronizeOff() def connectCallBack(self, _oMethod): self.synchronizeOn() if (_oMethod != None): self.__edSlotCallBack.connect(_oMethod) self.synchronizeOff() def getJobId(self): """ @return: JobId i.e. EDPluginName-Number @rtype: string """ return self.__jobId def getPluginName(self): """ @return: Name of the plugin @rtype: string """ return self.__strPluginName def getPlugin(self): """ @return: the plugin (instance) @rtype: python object """ return self.__edPlugin def getStatus(self): """ @return: status of the Job @rtype: string """ return self.__status def getMemSize(self): """ try to guess the size in memory of a job @return: expected size in memory """ if asizeof is not None: return asizeof.asizeof(self) @staticmethod def getStatusFromID(jobId): """ Retrieve the job (hence the plugin) status @param jobId: the Job identification number @type jobId: string @return: the EDJob status @rtype: string """ if jobId in EDJob.__dictJobs: return EDJob.__dictJobs[jobId].getStatus() else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) @staticmethod def getJobFromID(jobId): """ Retrieve the job (hence the plugin) @param jobId: the Job identification number @type jobId: string @return: the "EDJob instance", which contains the plugin (__edPlugin) and the status @rtype: a Python object. """ if jobId in EDJob.__dictJobs: return EDJob.__dictJobs[jobId] else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) @staticmethod def getMemoryFootprint(): if asizeof is not None: return asizeof.asizesof(EDJob.__dictJobs) @staticmethod def stats(): """ retrieve some statistics """ output = [] fExecTime = time.time() - EDJob.__fStartTime keys = EDJob.__dictJobs.keys() for key in keys: num = int(key.split("-", 1)[1]) job = EDJob.__dictJobs[key] output.append([ num, job.getPluginName(), job.getStatus(), job.getPlugin().getRunTime(), job.getMemSize() ]) output.sort() iNbJob = max(1, len(output)) EDVerbose.screen("%s\t|\t%s\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("id", "EDPluginName", "status", "runtime", "memory")) fWall = 0.0 fSumProd = 0.0 fSumX = 0.0 fSumX2 = 0.0 for oneJob in output: fWall += oneJob[3] fSumX += oneJob[0] fSumX2 += oneJob[0] * oneJob[0] fSumProd += oneJob[0] * oneJob[3] EDVerbose.screen("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob)) EDVerbose.screen("_" * 90) EDVerbose.screen( "Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime)) EDVerbose.screen( "Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob)) fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX) fOrd = (fWall - fSlope * fSumX) / iNbJob EDVerbose.screen( "Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope))
class EDJob(EDLogging): """ Create a module called EDJob * Most of what was done up to 09-2010 in EDParallelExecute should be done here * Each instance will be a job * Constructor takes a plugin name * Each instance will have taking an "setDataInput" method getting an XMLin (as string) * Each instance will gave a "getDataOutput" method with optional join * there could be a "join" method, waiting for the job to finish * Each instance will have a "execute" method and returning a JobId * Each instance will have a "setCallBack" method that stores the name of the external callback * provide status of a job * keep track of all plugin status * leave the time to plugin to initialize * static class retrieve job-instance, status, small-log ... * prevent multiple run of a single job ? * does not manage workload of the computer, should be managed at the ExecPlugin level Used for the tango binding, EDParallelExecute ... == class variables == dictPluginStatus[pluginName] = ["uninitialized"|"running"|"executed"| "failed"] dictJobs [JobId] = EDJob.Instance == static methods == getJob(JobId) """ PLUGIN_STATE_UNITIALIZED = "uninitialized" PLUGIN_STATE_RUNNING = "running" PLUGIN_STATE_SUCCESS = "success" PLUGIN_STATE_FAILURE = "failure" __edFactoryPlugin = EDFactoryPlugin() __dictJobs = {} __semaphore = Semaphore() __fStartTime = time.time() def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDLogging.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__pathXSDInput = None self.__pathXSDOutput = None self.__bXmlInputSet = False self.__status = None self.__name = None self.__runtime = None self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin( self.__strPluginName) if self.__edPlugin is None: raise RuntimeError("Unable to create plugin %s" % self.__strPluginName) self.__jobId = "%s-%08i" % (self.__strPluginName, self.__edPlugin.getId()) with self.__class__.__semaphore: self.__class__.__dictJobs[self.__jobId] = self if (self.__edPlugin is None): self.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the job (plugin) input data. @param: _oDataInput: could be either an String XML or an XSData object. @param _strDataInputKey: the key of an input data dictionnary The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ if _oDataInput in ["", None]: self.__bXmlInputSet = False return else: with self.locked(): if (self.__edPlugin is not None): self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey) self.__bXmlInputSet = True else: self.WARNING( "Setting DataInput for uninstanciated plugin %s." % self.__strPluginName) def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ if (self.__edPlugin is not None): return self.__edPlugin.getDataInput(_strDataInputKey).marshal() elif (self.__pathXSDInput is not None): return open(self.__pathXSDInput).read() else: self.WARNING("Getting DataInput for uninstanciated plugin %s." % self.__strPluginName) dataInput = property(getDataInput, setDataInput) def getDataOutput(self, _strDataOutputKey=None, _bWait=True): """ Returns the Plugin Output Data @param _bWait: shall we wait for the plugin to finish to retrieve output data: Yes by default. @type _bWait: boolean """ if _bWait: # Wait for plugin to finish befor returning data output self.synchronize() if (self.__edPlugin is not None): return self.__edPlugin.getDataOutput(_strDataOutputKey).marshal() elif self.__pathXSDOutput is not None: return open(self.__pathXSDOutput).read() else: self.WARNING( "Getting DataOutput for uninstanciated plugin or plugin has been garbage collected or output data were not saved. JobID was %s ." % self.__jobId) dataOutput = property(getDataOutput) def execute(self): """ Launch the EDNA plugin @return: JobId @rtype: string """ if not self.__bXmlInputSet: self.WARNING("Not executing job %s as input is empty" % self.__jobId) if (self.__edPlugin is not None): with self.locked(): self.__edPlugin.connectSUCCESS(self.successPluginExecution) self.__edPlugin.connectFAILURE(self.failurePluginExecution) self.__status = EDJob.PLUGIN_STATE_RUNNING self.__edPlugin.execute() return self.__jobId else: self.WARNING("Trying to run a plugin that does not exist: %s " % self.__strPluginName) def synchronize(self): """ Synchronize the execution of the job with the calling thread. """ with self.locked(): strStatus = self.__status if strStatus == EDJob.PLUGIN_STATE_RUNNING: self.__edPlugin.synchronize() elif strStatus == EDJob.PLUGIN_STATE_UNITIALIZED: self.WARNING("Unable to synchronize %s jobs" % strStatus) else: self.DEBUG("Unable to synchronize %s jobs" % strStatus) @classmethod def synchronizeAll(cls): """ Wait for all jobs to finish. """ EDVerbose.DEBUG("EDJob.synchronizeAll class method ") listJob = cls.__dictJobs.keys() for jobid in listJob: job = cls.__dictJobs[jobid] job.synchronize() if len(cls.__dictJobs) != len(listJob): EDVerbose.WARNING( "EDJob.synchronizeAll: New jobs have been launched while synchronizing" ) def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ with self.locked(): self.__status = EDJob.PLUGIN_STATE_SUCCESS self.screen("Plugin %s: success after %.3fs" % (self.__jobId, _edObject.getRunTime())) try: self.__edSlotSUCCESS.call(self.__jobId) except Exception: self.ERROR("Error in execution of Success call-back for %s" % self.__jobId) self.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: self.ERROR( "Error in execution of Common call-back (after success) for %s" % self.__jobId) self.writeErrorTrace() def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ with self.locked(): self.__status = EDJob.PLUGIN_STATE_FAILURE self.screen("Plugin %s: failure after %.3fs" % (self.__jobId, _edObject.getRunTime())) try: self.__edSlotFAILURE.call(self.__jobId) except Exception: self.ERROR("Error in execution of Failure call-back for %s" % self.__jobId) self.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: self.ERROR( "Error in execution of Common call-back (after failure) for %s" % self.__jobId) self.writeErrorTrace() def connectSUCCESS(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotSUCCESS.connect(_oMethod) def connectFAILURE(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotFAILURE.connect(_oMethod) def connectCallBack(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotCallBack.connect(_oMethod) def getJobId(self): """ @return: JobId i.e. EDPluginName-Number @rtype: string """ return self.__jobId jobId = property(getJobId, "EDJob.jobId: read-only property") getJobID = getJobId def getPluginName(self): """ @return: Name of the plugin @rtype: string """ return self.__strPluginName pluginName = property(getPluginName, "EDJob.pluginName: read-only property") def getPlugin(self): """ @return: the plugin (instance) @rtype: python object """ return self.__edPlugin plugin = property(getPlugin, "EDJob.plugin: read-only property") def getStatus(self): """ @return: status of the Job @rtype: string """ return self.__status status = property(getStatus, "EDJob.status: read-only property") def getName(self): return self.__name def setName(self, _strName): if self.__name is None: self.__name = _strName else: self.WARNING("EDJob.setName: One cannot rename a Job !!!") name = property(getName, setName, "EDJob.name: nickname of the job") def getMemSize(self): """ try to guess the size in memory of a job @return: expected size in memory """ if asizeof is not None: return asizeof.asizeof(self) @classmethod def getStatusFromID(cls, jobId): """ Retrieve the job (hence the plugin) status @param jobId: the Job identification number @type jobId: string @return: the EDJob status @rtype: string """ if jobId in cls.__dictJobs: strRet = cls.__dictJobs[jobId].getStatus() else: strRet = "Unable to retrieve such job: %s" % jobId EDVerbose.WARNING(strRet) return strRet getStatusFromId = getStatusFromID @classmethod def getJobFromID(cls, jobId): """ Retrieve the job (hence the plugin) @param jobId: the Job identification number @type jobId: string @return: the "EDJob instance", which contains the plugin (__edPlugin) and the status @rtype: a Python object. """ if jobId in cls.__dictJobs: return cls.__dictJobs[jobId] else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) getJobFromId = getJobFromID @classmethod def getMemoryFootprint(cls): if asizeof is not None: return asizeof.asizesof(cls.__dictJobs) def cleanJob(self, forceGC=True): """ Frees the memory associated with the top level plugin @param forceGC: Force garbage collection after clean-up @type forceGC: boolean """ self.synchronize() with self.locked(): if self.__edPlugin is not None: self.__pathXSDOutput = self.__edPlugin.strPathDataOutput self.__pathXSDInput = self.__edPlugin.strPathDataInput self.__runtime = self.__edPlugin.getRunTime() self.__edPlugin = None if forceGC: gc.collect() @classmethod def cleanJobfromId(cls, jobId, forceGC=True): """ Frees the memory associated with the top level plugin @param jobId: the Job identification number @type jobId: string @param forceGC: Force garbage collection after clean-up @type forceGC: boolean """ if jobId in cls.__dictJobs: job = cls.__dictJobs[jobId] job.cleanJob(forceGC) strRet = "Job %s cleaned" % jobId else: strRet = "Unable to retrieve such EDJob: %s" % jobId EDVerbose.WARNING(strRet) return strRet cleanJobfromID = cleanJobfromId @classmethod def getDataOutputFromId(cls, jobId): """ Returns the Plugin Output Data @param jobId: job idenfier @type jobId: string @return: edJob.DataOutput XML string """ output = None job = cls.getJobFromId(jobId) if job is not None: output = job.getDataOutput() return output or "" getDataOutputFromID = getDataOutputFromId @classmethod def getDataInputFromId(cls, jobId): """ Returns the Plugin Input Data @param jobId: job idenfier @type jobId: string @return: edJob.DataInput XML string """ output = None job = cls.getJobFromId(jobId) if job is not None: output = job.getDataInput() return output or "" getDataInputFromID = getDataInputFromId @classmethod def countRunning(cls): """ return the number of jobs still running. """ running = 0 for jobid in list(cls.__dictJobs.keys()): # python3 ! running += ( cls.__dictJobs[jobid].__status == cls.PLUGIN_STATE_RUNNING) return running @classmethod def stats(cls): """ Retrieve some statistics and print them """ lstStrOut = [] output = [] fExecTime = time.time() - cls.__fStartTime keys = cls.__dictJobs.keys() keys.sort() for num, key in enumerate(keys): job = cls.__dictJobs[key] if job.getPlugin() is None: runtime = job.__runtime else: runtime = job.getPlugin().getRunTime() output.append( [num, key, job.getStatus(), runtime, job.getMemSize()]) output.sort() iNbJob = max(1, len(keys)) lstStrOut.append("_" * 110) lstStrOut.append( "%s\t|\t%s\t\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("nr", "EDPluginName-Id", "status", "runtime", "memory")) lstStrOut.append("_" * 110) fWall = 0.0 fSumProd = 0.0 fSumX = 0.0 fSumX2 = 0.0 for oneJob in output: fWall += oneJob[3] fSumX += oneJob[0] fSumX2 += oneJob[0] * oneJob[0] fSumProd += oneJob[0] * oneJob[3] lstStrOut.append("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob)) lstStrOut.append("_" * 110) lstStrOut.append( "Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime)) lstStrOut.append( "Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob)) if len(keys) > 1: fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX) fOrd = (fWall - fSlope * fSumX) / iNbJob else: fSlope = 0.0 fOrd = fWall lstStrOut.append( "Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope)) strOutput = os.linesep.join(lstStrOut) EDVerbose.screen(strOutput) return strOutput
class EDParallelExecute: """ A class helping to make a multi-threaded application from a plugin name and a list of files. """ __edFactoryPlugin = EDFactoryPlugin() def __init__(self, _strPluginName, _functXMLin, \ _functXMLout=None, _functXMLerr=None, \ _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None): """ This is the constructor of the edna plugin launcher. @param _strPluginName: the name of the ENDA plugin @type _strPluginName: python string @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. @type _functXMLin: python function @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout @type _functXMLOut: python function @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin @type _functXMLErr: python function @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected. @type _iNbThreads: python integer @param _fDelay: The delay in seconds between two directories analysis @type _fDelay: python float @param _bVerbose: Do you want the EDNA plugin execution to be verbose ? @type _bVerbose: boolean @param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ? @type _bDebug: boolean """ self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads) EDUtilsParallel.initializeNbThread(self.__iNbThreads) self.__semaphoreOut = threading.Semaphore() self.__semaphoreErr = threading.Semaphore() self.__strPluginName = _strPluginName self.__functXMLin = _functXMLin self.__functXMLout = _functXMLout self.__functXMLerr = _functXMLerr self.__strCurrWorkDir = os.getcwd() self.__strTempDir = None self.__listInputPaths = [] if _bVerbose is not None: if _bVerbose: EDVerbose.setVerboseDebugOn() else: EDVerbose.setVerboseOff() if _bDebug is not None: if _bDebug: EDVerbose.setVerboseDebugOn() else: EDVerbose.setVerboseDebugOff() self.__fDelay = _fDelay #default delay between two directory checks. self.__bQuit = False # To check if we should quit the application self.__bIsFirstExecute = True signal.signal(signal.SIGTERM, self.handleKill) signal.signal(signal.SIGINT, self.handleKill) def runEDNA(self, _pyListInputPaths=["."], _strMode="dirwatch", _bNewerOnly=False): """ This method runs the parallel execution on the list of directories. @param _pyListInputPaths: the name of the directories to look after. @type _pyListInputPaths: python list of strings @param _strMode: can be dirwatch, inotify, or OffLine (inotify being not yet implemented) @type _strMode: python string @param _bNewerOnly: in online mode, process only new files (appearing after the program has started), by default it will process all files then wait for newer files and process them. @type _bNewerOnly: boolean """ self.moveToTempDir() self.__listInputPaths = _pyListInputPaths # if _strMode == "dirwatch": # self.watch_directories(_bNewerOnly) # elif _strMode == "inotify": # print "inotify online notify mode not yet implemented" # raise # else: #mode offline self.runEdnaFunction(self.__listInputPaths, _bIncludeSubdirs=True) self.waitForAllProcessToFinish() def moveToTempDir(self): """ Create a temporary directory and put all logs there """ self.__strCurrWorkDir = os.getcwd() self.__strTempDir = tempfile.mkdtemp(suffix='.log', prefix='edna-') EDVerbose.screen("The log directory of EDNA will be in " + self.__strTempDir) os.chdir(self.__strTempDir) def start(self, _strXmlInput): """ Launch EDNA with the given XML stream @param _strXmlInput: XML to be passed to the plugin @type _strXmlInput: python string representing the XML data structure """ if (_strXmlInput is None) or (_strXmlInput == ""): return #This is a trick to work-around bug #463: # Run the fist thread alone (delay the second, third, ...) # semaphore._Semaphore__value is the current value of the value, unfortunatly it is a protected value without getter # I need the value of the semaphore to guess if the current call is the first or not. # Nota semaphore are decreased in value from self.__iNbThreads to 0. When Zero, the semaphore is blocking. # Them all other, limited by the semaphore. if self.__bIsFirstExecute: sys.stdout.write("Waiting for first thread to initialize ....") while EDUtilsParallel.getNbRunning() > 0: time.sleep(0.5) sys.stdout.write(".") EDUtilsParallel.semaphoreNbThreadsAcquire() edPlugin = EDParallelExecute.__edFactoryPlugin.loadPlugin( self.__strPluginName) if (edPlugin is not None): edPlugin.setDataInput(_strXmlInput) edPlugin.connectSUCCESS(self.successPluginExecution) edPlugin.connectFAILURE(self.failurePluginExecution) edPlugin.execute() #edPlugin.executeSynchronous() else: EDUtilsParallel.semaphoreNbThreadsRelease() self.__bIsFirstExecute = False EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName) def runEdnaFunction(self, _listNewFiles, _bIncludeSubdirs=False): """ This method is the launcher for new files found by watch_directories ; it is also called directly in offline mode. @param _listNewFiles: list of files newly created in the directory. @type _listNewFiles: python list of strings. @param _bIncludeSubdirs: should we include sub-directories ? yes for offline and no for online. @type _bIncludeSubdirs: boolean """ dictListFiles = {} for oneFile in _listNewFiles: if os.path.isfile(oneFile): if self.__bQuit == True: return base = list(os.path.splitext(oneFile)[0]) base.reverse() for i in xrange(len(base)): if not base[i].isdigit(): break base = base[i:] base.reverse() base = "".join(base) if base in dictListFiles: dictListFiles[base].append(oneFile) else: dictListFiles[base] = [oneFile] for key in dictListFiles: EDVerbose.screen("Processing %s: %s" % (key, dictListFiles[key])) self.start(self.__functXMLin(dictListFiles[key])) def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ EDUtilsParallel.semaphoreNbThreadsRelease() self.__bIsFirstExecute = False if self.__functXMLout is None: EDVerbose.screen("Plugin %s execution ended with success" % self.__strPluginName) else: self.__semaphoreOut.acquire() self.__functXMLout(_edObject.dataInput.marshal(), _edObject.getDataOutput().marshal()) self.__semaphoreOut.release() def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ EDUtilsParallel.semaphoreNbThreadsRelease() self.__bIsFirstExecute = False if self.__functXMLerr is None: EDVerbose.screen("Plugin %s execution ended with failure" % self.__strPluginName) else: self.__semaphoreErr.acquire() self.__functXMLerr(_edObject.dataInput.marshal()) self.__semaphoreErr.release() def unlockErrFunction(self): """Method to unlock the semaphore that controls the call of an external procedure (procedure called in case of error in the EDNA pipeline)""" self.__semaphoreErr.release() def unlockOutFunction(self): """Method to unlock the semaphore that controls the call of an external procedure (procedure called in case of success of EDNA pipeline)""" self.__semaphoreOut.release() def lockErrFunction(self): """Method to lock the semaphore that controls the call of an external procedure (procedure called in case of error in the EDNA pipeline)""" self.__semaphoreErr.acquire() def lockOutFunction(self): """Method to lock the semaphore that controls the call of an external procedure (procedure called in case of success of the EDNA pipeline)""" self.__semaphoreOut.acquire() def semaphoreNbThreadsAcquire(self): """Method to acquire the semaphore that controls the number of threads running concurrently""" EDUtilsParallel.semaphoreNbThreadsAcquire() def semaphoreNbThreadsRelease(self): """Method to release the semaphore that controls the number of threads running concurrently""" EDUtilsParallel.semaphoreNbThreadsRelease() def handleKill(self, signum, frame): """ This method is launched when the program catches ctrl-c or get killed. It initialize the exit of the program """ self.__bQuit = True sys.stderr.write("Exit requested by signal %s with frame %s.\n" % (signum, frame)) self.waitForAllProcessToFinish() os.chdir(self.__strCurrWorkDir) def flush(self): """ This method calls the functXMLin a few times with a flush=True argument or without arguments and finishes the work """ bFinished = False while not bFinished: xml = None try: xml = self.__functXMLin(None, flush=True) except TypeError: try: xml = self.__functXMLin("", flush=True) except TypeError: try: xml = self.__functXMLin("") except TypeError: try: xml = self.__functXMLin("") except TypeError: xml = None if (xml is None) or (xml == ""): bFinished = True else: EDVerbose.screen("Flushing data ...") self.start(xml) def waitForAllProcessToFinish(self): """ as it names says, this method waits for all plug-ins which are currently running to finish before returning. """ sys.stderr.write("Waiting for launched jobs to finish .") while (EDUtilsParallel.getNbRunning() > 0): time.sleep(1) sys.stderr.write(".") sys.stderr.write("Done.\n") def cleanUp(self, listMethods=[]): """ Final hook if you need to execute something after all processes finished (like killAllWorkers in SPD) @param listMethods: allows to finish some things in the plugin. @type listMethods: list of strings representing names of methods of the plugin to be called. """ self.waitForAllProcessToFinish() edPlugin = EDParallelExecute.__edFactoryPlugin.loadPlugin( self.__strPluginName) for strOneMethod in listMethods: try: print "calling edPlugin.%s" % strOneMethod exec "edPlugin.%s" % strOneMethod except Exception: print "error in processing " + strOneMethod
def testSaveModuleDictionaryToDisk(self): edFactoryPlugin = EDFactoryPlugin() edPluginTest = edFactoryPlugin.loadPlugin("EDPluginTestPluginFactory") edFactoryPlugin.saveModuleDictionaryToDisk("testDictionary.xml")
def testLoadPlugin(self): edFactoryPlugin = EDFactoryPlugin() edPluginTest = edFactoryPlugin.loadPlugin("EDPluginTestPluginFactory") EDAssert.equal("TestReturnValue", edPluginTest.getTestValue())
def testGetProjectName(self): edFactoryPlugin = EDFactoryPlugin() strProjectName = edFactoryPlugin.getProjectName("EDPluginTestPluginFactory") EDAssert.equal("testProject", strProjectName)
def testGetProjectName(self): edFactoryPlugin = EDFactoryPlugin() strProjectName = edFactoryPlugin.getProjectName( "EDPluginTestPluginFactory") EDAssert.equal("testProject", strProjectName)
def testLoadModuleDictionaryFromDisk(self): edFactoryPlugin = EDFactoryPlugin() edFactoryPlugin.loadModuleDictionaryFromDisk("testDictionary.xml")