def appendPath(self, _strPluginTestLocation):
"""
For the tests, both the plugin directory and its corresponding "src" directory must
be on the python path (see the class documentation for EDFactoryPluginTest).
This method appends the plugin "src" directory to the system path, if it's not already present.
@param _strModuleLocation: Path to the module location
@type _strModuleLocation: python string
"""
EDFactoryPlugin.appendPath(self, _strPluginTestLocation)
strSrcDirectory = EDUtilsPath.appendListOfPaths(
_strPluginTestLocation, ["..", "..", "..", "..", "src"])
if os.path.exists(strSrcDirectory):
if (not strSrcDirectory in sys.path):
sys.path.append(strSrcDirectory)
strSrcDirectory = EDUtilsPath.appendListOfPaths(
_strPluginTestLocation, ["..", "..", "src"])
if os.path.exists(strSrcDirectory):
if (not strSrcDirectory in sys.path):
sys.path.append(strSrcDirectory)
strPluginDirectory = EDUtilsPath.appendListOfPaths(
_strPluginTestLocation, ["..", "..", "plugins"])
if os.path.exists(strPluginDirectory):
if (not strPluginDirectory in sys.path):
sys.path.append(strPluginDirectory)
def appendPath(self, _strPluginTestLocation):
"""
For the tests, both the plugin directory and its corresponding "src" directory must
be on the python path (see the class documentation for EDFactoryPluginTest).
This method appends the plugin "src" directory to the system path, if it's not already present.
@param _strModuleLocation: Path to the module location
@type _strModuleLocation: python string
"""
EDFactoryPlugin.appendPath(self, _strPluginTestLocation)
strSrcDirectory = EDUtilsPath.appendListOfPaths(_strPluginTestLocation, [ "..", "..", "..", "..", "src" ])
if os.path.exists(strSrcDirectory):
if (not strSrcDirectory in sys.path):
sys.path.append(strSrcDirectory)
strSrcDirectory = EDUtilsPath.appendListOfPaths(_strPluginTestLocation, [ "..", "..", "src" ])
if os.path.exists(strSrcDirectory):
if (not strSrcDirectory in sys.path):
sys.path.append(strSrcDirectory)
strPluginDirectory = EDUtilsPath.appendListOfPaths(_strPluginTestLocation, [ "..", "..", "plugins" ])
if os.path.exists(strPluginDirectory):
if (not strPluginDirectory in sys.path):
sys.path.append(strPluginDirectory)
def testGetProjectRootDirectory(self):
edFactoryPlugin = EDFactoryPlugin()
strEdnaHome = EDUtilsPath.getEdnaHome()
strTestProjectRootDirectoryReference = EDUtilsPath.appendListOfPaths(strEdnaHome, [ "kernel", "tests", "data", "EDFactoryPlugin", "testProject" ])
strTestProjectRootDirectory1 = edFactoryPlugin.getProjectRootDirectory("EDPluginTestPluginFactory")
EDAssert.equal(strTestProjectRootDirectoryReference, strTestProjectRootDirectory1)
strTestProjectRootDirectory2 = edFactoryPlugin.getProjectRootDirectory("XSDataTestProject")
EDAssert.equal(strTestProjectRootDirectoryReference, strTestProjectRootDirectory2)
strTestProjectRootDirectory3 = edFactoryPlugin.getProjectRootDirectory("PluginThatNotExists")
EDAssert.equal(None, strTestProjectRootDirectory3)
def createPlugin(self):
"""
Creates a plugin instance
"""
edPlugin = None
exceptionObject = None
try:
edFactoryPlugin = EDFactoryPlugin()
edPlugin = edFactoryPlugin.loadPlugin(self.getPluginName())
except ImportError, exceptionObject:
strWarningMessage = "Could not create the plugin: %s, reason: %s" % (self.getPluginName(), exceptionObject)
EDVerbose.WARNING(strWarningMessage)
def testGetPathToProjectConfigurationFile(self):
EDUtilsPath.setEdnaSite("TestSite")
edFactoryPlugin = EDFactoryPlugin()
strPathToConfigurationFile1 = edFactoryPlugin.getPathToProjectConfigurationFile("EDPluginTestPluginFactory")
strPathToConfigurationFileReference1 = EDUtilsPath.appendListOfPaths(EDUtilsPath.getEdnaHome(),
[ "edna-kernel", "tests", "data", "EDFactoryPlugin", \
"testProject", "conf", "XSConfiguration_TestSite.xml" ])
EDAssert.equal(strPathToConfigurationFileReference1, strPathToConfigurationFile1)
EDUtilsPath.setEdnaSite("NonexistingTestSite")
strPathToConfigurationFile2 = edFactoryPlugin.getPathToProjectConfigurationFile("EDPluginTestPluginFactory")
strPathToConfigurationFileReference2 = None
EDAssert.equal(strPathToConfigurationFileReference2, strPathToConfigurationFile2)
# Since this is a static variable we need to reset it to None in order not to break any other tests...
EDUtilsPath.setEdnaSite(None)
def createPlugin(self):
"""
Creates a plugin instance
"""
edPlugin = None
exceptionObject = None
try:
edFactoryPlugin = EDFactoryPlugin()
edPlugin = edFactoryPlugin.loadPlugin(self.getPluginName())
except ImportError(exceptionObject):
strWarningMessage = "Could not create the plugin: %s, reason: %s" % (self.getPluginName(), exceptionObject)
EDVerbose.WARNING(strWarningMessage)
if edPlugin is None:
if exceptionObject is None:
EDVerbose.error("EDTestCasePlugin.createPlugin: Could not create plugin: " + self.getPluginName())
return edPlugin
def testGetProjectRootDirectory(self):
edFactoryPlugin = EDFactoryPlugin()
strEdnaHome = EDUtilsPath.getEdnaHome()
strTestProjectRootDirectoryReference = EDUtilsPath.appendListOfPaths(
strEdnaHome,
["kernel", "tests", "data", "EDFactoryPlugin", "testProject"])
strTestProjectRootDirectory1 = edFactoryPlugin.getProjectRootDirectory(
"EDPluginTestPluginFactory")
EDAssert.equal(strTestProjectRootDirectoryReference,
strTestProjectRootDirectory1)
strTestProjectRootDirectory2 = edFactoryPlugin.getProjectRootDirectory(
"XSDataTestProject")
EDAssert.equal(strTestProjectRootDirectoryReference,
strTestProjectRootDirectory2)
strTestProjectRootDirectory3 = edFactoryPlugin.getProjectRootDirectory(
"PluginThatNotExists")
EDAssert.equal(None, strTestProjectRootDirectory3)
def createPlugin(self):
"""
Creates a plugin instance
"""
edPlugin = None
exceptionObject = None
try:
edFactoryPlugin = EDFactoryPlugin()
edPlugin = edFactoryPlugin.loadPlugin(self.getPluginName())
except ImportError(exceptionObject):
strWarningMessage = "Could not create the plugin: %s, reason: %s" % (
self.getPluginName(), exceptionObject)
EDVerbose.WARNING(strWarningMessage)
if edPlugin is None:
if exceptionObject is None:
EDVerbose.error(
"EDTestCasePlugin.createPlugin: Could not create plugin: "
+ self.getPluginName())
return edPlugin
class EDJob(EDObject):
PLUGIN_STATE_UNITIALIZED = "uninitialized"
PLUGIN_STATE_RUNNING = "running"
PLUGIN_STATE_SUCCESS = "success"
PLUGIN_STATE_FAILURE = "failure"
__edFactoryPlugin = EDFactoryPlugin()
__dictJobs = {}
__semaphore = threading.Semaphore()
__dictPluginLastId = {}
__fStartTime = time.time()
def __init__(self, _strPluginName):
"""
Constructor of the class
@param strPluginName: name of the plugin
@type strPluginName: string
"""
EDObject.__init__(self)
self.__strPluginName = _strPluginName
self.__edPlugin = None
self.__edSlotCallBack = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__bXmlInputSet = False
self.__status = None
# self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName)
self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName)
EDJob.__semaphore.acquire()
# Create the JobID
if not self.__strPluginName in EDJob.__dictPluginLastId:
EDJob.__dictPluginLastId[_strPluginName] = 0
else:
EDJob.__dictPluginLastId[_strPluginName] += 1
self.__jobId = "%s-%i" % (self.__strPluginName,
EDJob.__dictPluginLastId[_strPluginName])
EDJob.__dictJobs[self.__jobId] = self
EDJob.__semaphore.release()
if (self.__edPlugin is None):
EDVerbose.WARNING("Instantiation of plugin %s failed!!!" %
_strPluginName)
else:
self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def setDataInput(self, _oDataInput, _strDataInputKey=None):
"""
Sets the job (plugin) input data.
@param: _oDataInput: could be either an String XML or an XSData object.
@param _strDataInputKey: the key of an input data dictionnary
The input data is stored in a dictionary with the key _strDataInputKey.
If the key is not provided a default key is used.
If not data input class is defined for the key an exception is raised.
If the key is not the default key, the data object is added to a list which
might contain already stored object(s).
If _oDataInput is None the list corresponding to a keyword is deleted.
"""
if _oDataInput in ["", None]:
self.__bXmlInputSet = False
return
self.synchronizeOn()
if (self.__edPlugin is not None):
self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey)
self.__bXmlInputSet = True
else:
EDVerbose.WARNING(
"Setting DataInput for uninstanciated plugin %s." %
self.__strPluginName)
self.synchronizeOff()
def getDataInput(self, _strDataInputKey=None):
"""
Returns the Plugin Input Data for a particular key.
If the key is not provided a default key is used.
"""
if (self.__edPlugin is not None):
return self.__edPlugin.getDataInput(_strDataInputKey)
else:
EDVerbose.WARNING(
"Getting DataInput for uninstanciated plugin %s." %
self.__strPluginName)
def getDataOutput(self, _strDataOutputKey=None):
"""
Returns the Plugin Output Data
"""
if (self.__edPlugin is not None):
return self.__edPlugin.getDataOutput(_strDataOutputKey)
else:
EDVerbose.WARNING(
"Getting DataOutput for uninstanciated plugin %s." %
self.__strPluginName)
def execute(self):
"""
Launch the EDNA plugin
@return: JobId
@rtype: string
"""
returnId = None
if not self.__bXmlInputSet:
EDVerbose.WARNING("Not executing job %s as input is empty" %
self.__jobId)
if (self.__edPlugin is not None):
self.synchronizeOn()
self.__status = EDJob.PLUGIN_STATE_RUNNING
self.__edPlugin.connectSUCCESS(self.successPluginExecution)
self.__edPlugin.connectFAILURE(self.failurePluginExecution)
self.__edPlugin.execute()
returnId = self.__jobId
self.synchronizeOff()
else:
EDVerbose.WARNING(
"Trying to run a plugin that does not exist: %s " %
self.__strPluginName)
return returnId
def synchronize(self):
"""
Synchronize the execution of the job with the calling thread.
"""
self.__edPlugin.synchronize()
def successPluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin succeeds
"""
self.synchronizeOn()
self.__status = EDJob.PLUGIN_STATE_SUCCESS
EDVerbose.screen("Plugin %s execution ended with success" %
self.__jobId)
self.synchronizeOff()
try:
self.__edSlotSUCCESS.call(self.__jobId)
except Exception:
EDVerbose.ERROR("Error in execution of Success call-back for %s" %
self.__jobId)
EDVerbose.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
EDVerbose.ERROR(
"Error in execution of Common call-back (after success) for %s"
% self.__jobId)
EDVerbose.writeErrorTrace()
def failurePluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin failed
"""
self.synchronizeOn()
self.__status = EDJob.PLUGIN_STATE_FAILURE
EDVerbose.screen("Plugin %s execution ended with failure" %
self.__jobId)
self.synchronizeOff()
try:
self.__edSlotFAILURE.call(self.__jobId)
except Exception:
EDVerbose.ERROR("Error in execution of Failure call-back for %s" %
self.__jobId)
EDVerbose.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
EDVerbose.ERROR(
"Error in execution of Common call-back (after failure) for %s"
% self.__jobId)
EDVerbose.writeErrorTrace()
def connectSUCCESS(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotSUCCESS.connect(_oMethod)
self.synchronizeOff()
def connectFAILURE(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotFAILURE.connect(_oMethod)
self.synchronizeOff()
def connectCallBack(self, _oMethod):
self.synchronizeOn()
if (_oMethod != None):
self.__edSlotCallBack.connect(_oMethod)
self.synchronizeOff()
def getJobId(self):
"""
@return: JobId i.e. EDPluginName-Number
@rtype: string
"""
return self.__jobId
def getPluginName(self):
"""
@return: Name of the plugin
@rtype: string
"""
return self.__strPluginName
def getPlugin(self):
"""
@return: the plugin (instance)
@rtype: python object
"""
return self.__edPlugin
def getStatus(self):
"""
@return: status of the Job
@rtype: string
"""
return self.__status
def getMemSize(self):
"""
try to guess the size in memory of a job
@return: expected size in memory
"""
if asizeof is not None:
return asizeof.asizeof(self)
@staticmethod
def getStatusFromID(jobId):
"""
Retrieve the job (hence the plugin) status
@param jobId: the Job identification number
@type jobId: string
@return: the EDJob status
@rtype: string
"""
if jobId in EDJob.__dictJobs:
return EDJob.__dictJobs[jobId].getStatus()
else:
EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId)
@staticmethod
def getJobFromID(jobId):
"""
Retrieve the job (hence the plugin)
@param jobId: the Job identification number
@type jobId: string
@return: the "EDJob instance", which contains the plugin (__edPlugin) and the status
@rtype: a Python object.
"""
if jobId in EDJob.__dictJobs:
return EDJob.__dictJobs[jobId]
else:
EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId)
@staticmethod
def getMemoryFootprint():
if asizeof is not None:
return asizeof.asizesof(EDJob.__dictJobs)
@staticmethod
def stats():
"""
retrieve some statistics
"""
output = []
fExecTime = time.time() - EDJob.__fStartTime
keys = EDJob.__dictJobs.keys()
for key in keys:
num = int(key.split("-", 1)[1])
job = EDJob.__dictJobs[key]
output.append([
num,
job.getPluginName(),
job.getStatus(),
job.getPlugin().getRunTime(),
job.getMemSize()
])
output.sort()
iNbJob = max(1, len(output))
EDVerbose.screen("%s\t|\t%s\t\t\t|\t%s\t|\t%s\t\t|\t%s" %
("id", "EDPluginName", "status", "runtime", "memory"))
fWall = 0.0
fSumProd = 0.0
fSumX = 0.0
fSumX2 = 0.0
for oneJob in output:
fWall += oneJob[3]
fSumX += oneJob[0]
fSumX2 += oneJob[0] * oneJob[0]
fSumProd += oneJob[0] * oneJob[3]
EDVerbose.screen("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" %
tuple(oneJob))
EDVerbose.screen("_" * 90)
EDVerbose.screen(
"Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f"
% (fWall, fExecTime, fWall / fExecTime))
EDVerbose.screen(
"Average execution time (Wall/N): %.3fs, Average throughput: %.3fs"
% (fWall / iNbJob, fExecTime / iNbJob))
fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 -
fSumX * fSumX)
fOrd = (fWall - fSlope * fSumX) / iNbJob
EDVerbose.screen(
"Regression of execution time: ExecTime = %.3f + %f * NbJob" %
(fOrd, fSlope))
class EDJob(EDLogging):
"""
Create a module called EDJob
* Most of what was done up to 09-2010 in EDParallelExecute should be done here
* Each instance will be a job
* Constructor takes a plugin name
* Each instance will have taking an "setDataInput" method getting an XMLin (as string)
* Each instance will gave a "getDataOutput" method with optional join
* there could be a "join" method, waiting for the job to finish
* Each instance will have a "execute" method and returning a JobId
* Each instance will have a "setCallBack" method that stores the name of the external callback
* provide status of a job
* keep track of all plugin status
* leave the time to plugin to initialize
* static class retrieve job-instance, status, small-log ...
* prevent multiple run of a single job ?
* does not manage workload of the computer, should be managed at the ExecPlugin level
Used for the tango binding, EDParallelExecute ...
== class variables ==
dictPluginStatus[pluginName] = ["uninitialized"|"running"|"executed"| "failed"]
dictJobs [JobId] = EDJob.Instance
== static methods ==
getJob(JobId)
"""
PLUGIN_STATE_UNITIALIZED = "uninitialized"
PLUGIN_STATE_RUNNING = "running"
PLUGIN_STATE_SUCCESS = "success"
PLUGIN_STATE_FAILURE = "failure"
__edFactoryPlugin = EDFactoryPlugin()
__dictJobs = {}
__semaphore = Semaphore()
__fStartTime = time.time()
def __init__(self, _strPluginName):
"""
Constructor of the class
@param strPluginName: name of the plugin
@type strPluginName: string
"""
EDLogging.__init__(self)
self.__strPluginName = _strPluginName
self.__edPlugin = None
self.__edSlotCallBack = EDSlot()
self.__edSlotSUCCESS = EDSlot()
self.__edSlotFAILURE = EDSlot()
self.__pathXSDInput = None
self.__pathXSDOutput = None
self.__bXmlInputSet = False
self.__status = None
self.__name = None
self.__runtime = None
self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(
self.__strPluginName)
if self.__edPlugin is None:
raise RuntimeError("Unable to create plugin %s" %
self.__strPluginName)
self.__jobId = "%s-%08i" % (self.__strPluginName,
self.__edPlugin.getId())
with self.__class__.__semaphore:
self.__class__.__dictJobs[self.__jobId] = self
if (self.__edPlugin is None):
self.WARNING("Instantiation of plugin %s failed!!!" %
_strPluginName)
else:
self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def setDataInput(self, _oDataInput, _strDataInputKey=None):
"""
Sets the job (plugin) input data.
@param: _oDataInput: could be either an String XML or an XSData object.
@param _strDataInputKey: the key of an input data dictionnary
The input data is stored in a dictionary with the key _strDataInputKey.
If the key is not provided a default key is used.
If not data input class is defined for the key an exception is raised.
If the key is not the default key, the data object is added to a list which
might contain already stored object(s).
If _oDataInput is None the list corresponding to a keyword is deleted.
"""
if _oDataInput in ["", None]:
self.__bXmlInputSet = False
return
else:
with self.locked():
if (self.__edPlugin is not None):
self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey)
self.__bXmlInputSet = True
else:
self.WARNING(
"Setting DataInput for uninstanciated plugin %s." %
self.__strPluginName)
def getDataInput(self, _strDataInputKey=None):
"""
Returns the Plugin Input Data for a particular key.
If the key is not provided a default key is used.
"""
if (self.__edPlugin is not None):
return self.__edPlugin.getDataInput(_strDataInputKey).marshal()
elif (self.__pathXSDInput is not None):
return open(self.__pathXSDInput).read()
else:
self.WARNING("Getting DataInput for uninstanciated plugin %s." %
self.__strPluginName)
dataInput = property(getDataInput, setDataInput)
def getDataOutput(self, _strDataOutputKey=None, _bWait=True):
"""
Returns the Plugin Output Data
@param _bWait: shall we wait for the plugin to finish to retrieve output data: Yes by default.
@type _bWait: boolean
"""
if _bWait: # Wait for plugin to finish befor returning data output
self.synchronize()
if (self.__edPlugin is not None):
return self.__edPlugin.getDataOutput(_strDataOutputKey).marshal()
elif self.__pathXSDOutput is not None:
return open(self.__pathXSDOutput).read()
else:
self.WARNING(
"Getting DataOutput for uninstanciated plugin or plugin has been garbage collected or output data were not saved. JobID was %s ."
% self.__jobId)
dataOutput = property(getDataOutput)
def execute(self):
"""
Launch the EDNA plugin
@return: JobId
@rtype: string
"""
if not self.__bXmlInputSet:
self.WARNING("Not executing job %s as input is empty" %
self.__jobId)
if (self.__edPlugin is not None):
with self.locked():
self.__edPlugin.connectSUCCESS(self.successPluginExecution)
self.__edPlugin.connectFAILURE(self.failurePluginExecution)
self.__status = EDJob.PLUGIN_STATE_RUNNING
self.__edPlugin.execute()
return self.__jobId
else:
self.WARNING("Trying to run a plugin that does not exist: %s " %
self.__strPluginName)
def synchronize(self):
"""
Synchronize the execution of the job with the calling thread.
"""
with self.locked():
strStatus = self.__status
if strStatus == EDJob.PLUGIN_STATE_RUNNING:
self.__edPlugin.synchronize()
elif strStatus == EDJob.PLUGIN_STATE_UNITIALIZED:
self.WARNING("Unable to synchronize %s jobs" % strStatus)
else:
self.DEBUG("Unable to synchronize %s jobs" % strStatus)
@classmethod
def synchronizeAll(cls):
"""
Wait for all jobs to finish.
"""
EDVerbose.DEBUG("EDJob.synchronizeAll class method ")
listJob = cls.__dictJobs.keys()
for jobid in listJob:
job = cls.__dictJobs[jobid]
job.synchronize()
if len(cls.__dictJobs) != len(listJob):
EDVerbose.WARNING(
"EDJob.synchronizeAll: New jobs have been launched while synchronizing"
)
def successPluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin succeeds
"""
with self.locked():
self.__status = EDJob.PLUGIN_STATE_SUCCESS
self.screen("Plugin %s: success after %.3fs" %
(self.__jobId, _edObject.getRunTime()))
try:
self.__edSlotSUCCESS.call(self.__jobId)
except Exception:
self.ERROR("Error in execution of Success call-back for %s" %
self.__jobId)
self.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
self.ERROR(
"Error in execution of Common call-back (after success) for %s"
% self.__jobId)
self.writeErrorTrace()
def failurePluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin failed
"""
with self.locked():
self.__status = EDJob.PLUGIN_STATE_FAILURE
self.screen("Plugin %s: failure after %.3fs" %
(self.__jobId, _edObject.getRunTime()))
try:
self.__edSlotFAILURE.call(self.__jobId)
except Exception:
self.ERROR("Error in execution of Failure call-back for %s" %
self.__jobId)
self.writeErrorTrace()
try:
self.__edSlotCallBack.call(self.__jobId)
except Exception:
self.ERROR(
"Error in execution of Common call-back (after failure) for %s"
% self.__jobId)
self.writeErrorTrace()
def connectSUCCESS(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotSUCCESS.connect(_oMethod)
def connectFAILURE(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotFAILURE.connect(_oMethod)
def connectCallBack(self, _oMethod):
"""
@param _oMethod: function or method to be called - back
"""
with self.locked():
if (_oMethod != None):
self.__edSlotCallBack.connect(_oMethod)
def getJobId(self):
"""
@return: JobId i.e. EDPluginName-Number
@rtype: string
"""
return self.__jobId
jobId = property(getJobId, "EDJob.jobId: read-only property")
getJobID = getJobId
def getPluginName(self):
"""
@return: Name of the plugin
@rtype: string
"""
return self.__strPluginName
pluginName = property(getPluginName,
"EDJob.pluginName: read-only property")
def getPlugin(self):
"""
@return: the plugin (instance)
@rtype: python object
"""
return self.__edPlugin
plugin = property(getPlugin, "EDJob.plugin: read-only property")
def getStatus(self):
"""
@return: status of the Job
@rtype: string
"""
return self.__status
status = property(getStatus, "EDJob.status: read-only property")
def getName(self):
return self.__name
def setName(self, _strName):
if self.__name is None:
self.__name = _strName
else:
self.WARNING("EDJob.setName: One cannot rename a Job !!!")
name = property(getName, setName, "EDJob.name: nickname of the job")
def getMemSize(self):
"""
try to guess the size in memory of a job
@return: expected size in memory
"""
if asizeof is not None:
return asizeof.asizeof(self)
@classmethod
def getStatusFromID(cls, jobId):
"""
Retrieve the job (hence the plugin) status
@param jobId: the Job identification number
@type jobId: string
@return: the EDJob status
@rtype: string
"""
if jobId in cls.__dictJobs:
strRet = cls.__dictJobs[jobId].getStatus()
else:
strRet = "Unable to retrieve such job: %s" % jobId
EDVerbose.WARNING(strRet)
return strRet
getStatusFromId = getStatusFromID
@classmethod
def getJobFromID(cls, jobId):
"""
Retrieve the job (hence the plugin)
@param jobId: the Job identification number
@type jobId: string
@return: the "EDJob instance", which contains the plugin (__edPlugin) and the status
@rtype: a Python object.
"""
if jobId in cls.__dictJobs:
return cls.__dictJobs[jobId]
else:
EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId)
getJobFromId = getJobFromID
@classmethod
def getMemoryFootprint(cls):
if asizeof is not None:
return asizeof.asizesof(cls.__dictJobs)
def cleanJob(self, forceGC=True):
"""
Frees the memory associated with the top level plugin
@param forceGC: Force garbage collection after clean-up
@type forceGC: boolean
"""
self.synchronize()
with self.locked():
if self.__edPlugin is not None:
self.__pathXSDOutput = self.__edPlugin.strPathDataOutput
self.__pathXSDInput = self.__edPlugin.strPathDataInput
self.__runtime = self.__edPlugin.getRunTime()
self.__edPlugin = None
if forceGC:
gc.collect()
@classmethod
def cleanJobfromId(cls, jobId, forceGC=True):
"""
Frees the memory associated with the top level plugin
@param jobId: the Job identification number
@type jobId: string
@param forceGC: Force garbage collection after clean-up
@type forceGC: boolean
"""
if jobId in cls.__dictJobs:
job = cls.__dictJobs[jobId]
job.cleanJob(forceGC)
strRet = "Job %s cleaned" % jobId
else:
strRet = "Unable to retrieve such EDJob: %s" % jobId
EDVerbose.WARNING(strRet)
return strRet
cleanJobfromID = cleanJobfromId
@classmethod
def getDataOutputFromId(cls, jobId):
"""
Returns the Plugin Output Data
@param jobId: job idenfier
@type jobId: string
@return: edJob.DataOutput XML string
"""
output = None
job = cls.getJobFromId(jobId)
if job is not None:
output = job.getDataOutput()
return output or ""
getDataOutputFromID = getDataOutputFromId
@classmethod
def getDataInputFromId(cls, jobId):
"""
Returns the Plugin Input Data
@param jobId: job idenfier
@type jobId: string
@return: edJob.DataInput XML string
"""
output = None
job = cls.getJobFromId(jobId)
if job is not None:
output = job.getDataInput()
return output or ""
getDataInputFromID = getDataInputFromId
@classmethod
def countRunning(cls):
"""
return the number of jobs still running.
"""
running = 0
for jobid in list(cls.__dictJobs.keys()): # python3 !
running += (
cls.__dictJobs[jobid].__status == cls.PLUGIN_STATE_RUNNING)
return running
@classmethod
def stats(cls):
"""
Retrieve some statistics and print them
"""
lstStrOut = []
output = []
fExecTime = time.time() - cls.__fStartTime
keys = cls.__dictJobs.keys()
keys.sort()
for num, key in enumerate(keys):
job = cls.__dictJobs[key]
if job.getPlugin() is None:
runtime = job.__runtime
else:
runtime = job.getPlugin().getRunTime()
output.append(
[num, key,
job.getStatus(), runtime,
job.getMemSize()])
output.sort()
iNbJob = max(1, len(keys))
lstStrOut.append("_" * 110)
lstStrOut.append(
"%s\t|\t%s\t\t\t\t|\t%s\t|\t%s\t\t|\t%s" %
("nr", "EDPluginName-Id", "status", "runtime", "memory"))
lstStrOut.append("_" * 110)
fWall = 0.0
fSumProd = 0.0
fSumX = 0.0
fSumX2 = 0.0
for oneJob in output:
fWall += oneJob[3]
fSumX += oneJob[0]
fSumX2 += oneJob[0] * oneJob[0]
fSumProd += oneJob[0] * oneJob[3]
lstStrOut.append("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" %
tuple(oneJob))
lstStrOut.append("_" * 110)
lstStrOut.append(
"Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f"
% (fWall, fExecTime, fWall / fExecTime))
lstStrOut.append(
"Average execution time (Wall/N): %.3fs, Average throughput: %.3fs"
% (fWall / iNbJob, fExecTime / iNbJob))
if len(keys) > 1:
fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 -
fSumX * fSumX)
fOrd = (fWall - fSlope * fSumX) / iNbJob
else:
fSlope = 0.0
fOrd = fWall
lstStrOut.append(
"Regression of execution time: ExecTime = %.3f + %f * NbJob" %
(fOrd, fSlope))
strOutput = os.linesep.join(lstStrOut)
EDVerbose.screen(strOutput)
return strOutput
class EDParallelExecute:
"""
A class helping to make a multi-threaded application from a plugin name and a list of files.
"""
__edFactoryPlugin = EDFactoryPlugin()
def __init__(self, _strPluginName, _functXMLin, \
_functXMLout=None, _functXMLerr=None, \
_iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
"""
This is the constructor of the edna plugin launcher.
@param _strPluginName: the name of the ENDA plugin
@type _strPluginName: python string
@param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin.
@type _functXMLin: python function
@param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
@type _functXMLOut: python function
@param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
@type _functXMLErr: python function
@param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected.
@type _iNbThreads: python integer
@param _fDelay: The delay in seconds between two directories analysis
@type _fDelay: python float
@param _bVerbose: Do you want the EDNA plugin execution to be verbose ?
@type _bVerbose: boolean
@param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ?
@type _bDebug: boolean
"""
self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
EDUtilsParallel.initializeNbThread(self.__iNbThreads)
self.__semaphoreOut = threading.Semaphore()
self.__semaphoreErr = threading.Semaphore()
self.__strPluginName = _strPluginName
self.__functXMLin = _functXMLin
self.__functXMLout = _functXMLout
self.__functXMLerr = _functXMLerr
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = None
self.__listInputPaths = []
if _bVerbose is not None:
if _bVerbose:
EDVerbose.setVerboseDebugOn()
else:
EDVerbose.setVerboseOff()
if _bDebug is not None:
if _bDebug:
EDVerbose.setVerboseDebugOn()
else:
EDVerbose.setVerboseDebugOff()
self.__fDelay = _fDelay #default delay between two directory checks.
self.__bQuit = False # To check if we should quit the application
self.__bIsFirstExecute = True
signal.signal(signal.SIGTERM, self.handleKill)
signal.signal(signal.SIGINT, self.handleKill)
def runEDNA(self,
_pyListInputPaths=["."],
_strMode="dirwatch",
_bNewerOnly=False):
"""
This method runs the parallel execution on the list of directories.
@param _pyListInputPaths: the name of the directories to look after.
@type _pyListInputPaths: python list of strings
@param _strMode: can be dirwatch, inotify, or OffLine (inotify being not yet implemented)
@type _strMode: python string
@param _bNewerOnly: in online mode, process only new files (appearing after the program has started), by default it will process all files then wait for newer files and process them.
@type _bNewerOnly: boolean
"""
self.moveToTempDir()
self.__listInputPaths = _pyListInputPaths
# if _strMode == "dirwatch":
# self.watch_directories(_bNewerOnly)
# elif _strMode == "inotify":
# print "inotify online notify mode not yet implemented"
# raise
# else: #mode offline
self.runEdnaFunction(self.__listInputPaths, _bIncludeSubdirs=True)
self.waitForAllProcessToFinish()
def moveToTempDir(self):
"""
Create a temporary directory and put all logs there
"""
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = tempfile.mkdtemp(suffix='.log', prefix='edna-')
EDVerbose.screen("The log directory of EDNA will be in " +
self.__strTempDir)
os.chdir(self.__strTempDir)
def start(self, _strXmlInput):
"""
Launch EDNA with the given XML stream
@param _strXmlInput: XML to be passed to the plugin
@type _strXmlInput: python string representing the XML data structure
"""
if (_strXmlInput is None) or (_strXmlInput == ""):
return
#This is a trick to work-around bug #463:
# Run the fist thread alone (delay the second, third, ...)
# semaphore._Semaphore__value is the current value of the value, unfortunatly it is a protected value without getter
# I need the value of the semaphore to guess if the current call is the first or not.
# Nota semaphore are decreased in value from self.__iNbThreads to 0. When Zero, the semaphore is blocking.
# Them all other, limited by the semaphore.
if self.__bIsFirstExecute:
sys.stdout.write("Waiting for first thread to initialize ....")
while EDUtilsParallel.getNbRunning() > 0:
time.sleep(0.5)
sys.stdout.write(".")
EDUtilsParallel.semaphoreNbThreadsAcquire()
edPlugin = EDParallelExecute.__edFactoryPlugin.loadPlugin(
self.__strPluginName)
if (edPlugin is not None):
edPlugin.setDataInput(_strXmlInput)
edPlugin.connectSUCCESS(self.successPluginExecution)
edPlugin.connectFAILURE(self.failurePluginExecution)
edPlugin.execute()
#edPlugin.executeSynchronous()
else:
EDUtilsParallel.semaphoreNbThreadsRelease()
self.__bIsFirstExecute = False
EDVerbose.screen("ERROR! Plugin not found : " +
self.__strPluginName)
def runEdnaFunction(self, _listNewFiles, _bIncludeSubdirs=False):
"""
This method is the launcher for new files found by watch_directories ; it is also called directly in offline mode.
@param _listNewFiles: list of files newly created in the directory.
@type _listNewFiles: python list of strings.
@param _bIncludeSubdirs: should we include sub-directories ? yes for offline and no for online.
@type _bIncludeSubdirs: boolean
"""
dictListFiles = {}
for oneFile in _listNewFiles:
if os.path.isfile(oneFile):
if self.__bQuit == True:
return
base = list(os.path.splitext(oneFile)[0])
base.reverse()
for i in xrange(len(base)):
if not base[i].isdigit():
break
base = base[i:]
base.reverse()
base = "".join(base)
if base in dictListFiles:
dictListFiles[base].append(oneFile)
else:
dictListFiles[base] = [oneFile]
for key in dictListFiles:
EDVerbose.screen("Processing %s: %s" % (key, dictListFiles[key]))
self.start(self.__functXMLin(dictListFiles[key]))
def successPluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin succeeds
"""
EDUtilsParallel.semaphoreNbThreadsRelease()
self.__bIsFirstExecute = False
if self.__functXMLout is None:
EDVerbose.screen("Plugin %s execution ended with success" %
self.__strPluginName)
else:
self.__semaphoreOut.acquire()
self.__functXMLout(_edObject.dataInput.marshal(),
_edObject.getDataOutput().marshal())
self.__semaphoreOut.release()
def failurePluginExecution(self, _edObject=None):
"""
Method called when the execution of the plugin failed
"""
EDUtilsParallel.semaphoreNbThreadsRelease()
self.__bIsFirstExecute = False
if self.__functXMLerr is None:
EDVerbose.screen("Plugin %s execution ended with failure" %
self.__strPluginName)
else:
self.__semaphoreErr.acquire()
self.__functXMLerr(_edObject.dataInput.marshal())
self.__semaphoreErr.release()
def unlockErrFunction(self):
"""Method to unlock the semaphore that controls the call of an external procedure (procedure called in case of error in the EDNA pipeline)"""
self.__semaphoreErr.release()
def unlockOutFunction(self):
"""Method to unlock the semaphore that controls the call of an external procedure (procedure called in case of success of EDNA pipeline)"""
self.__semaphoreOut.release()
def lockErrFunction(self):
"""Method to lock the semaphore that controls the call of an external procedure (procedure called in case of error in the EDNA pipeline)"""
self.__semaphoreErr.acquire()
def lockOutFunction(self):
"""Method to lock the semaphore that controls the call of an external procedure (procedure called in case of success of the EDNA pipeline)"""
self.__semaphoreOut.acquire()
def semaphoreNbThreadsAcquire(self):
"""Method to acquire the semaphore that controls the number of threads running concurrently"""
EDUtilsParallel.semaphoreNbThreadsAcquire()
def semaphoreNbThreadsRelease(self):
"""Method to release the semaphore that controls the number of threads running concurrently"""
EDUtilsParallel.semaphoreNbThreadsRelease()
def handleKill(self, signum, frame):
"""
This method is launched when the program catches ctrl-c or get killed. It initialize the exit of the program
"""
self.__bQuit = True
sys.stderr.write("Exit requested by signal %s with frame %s.\n" %
(signum, frame))
self.waitForAllProcessToFinish()
os.chdir(self.__strCurrWorkDir)
def flush(self):
"""
This method calls the functXMLin a few times with a flush=True argument or without arguments and finishes the work
"""
bFinished = False
while not bFinished:
xml = None
try:
xml = self.__functXMLin(None, flush=True)
except TypeError:
try:
xml = self.__functXMLin("", flush=True)
except TypeError:
try:
xml = self.__functXMLin("")
except TypeError:
try:
xml = self.__functXMLin("")
except TypeError:
xml = None
if (xml is None) or (xml == ""):
bFinished = True
else:
EDVerbose.screen("Flushing data ...")
self.start(xml)
def waitForAllProcessToFinish(self):
"""
as it names says, this method waits for all plug-ins which are currently running to finish before returning.
"""
sys.stderr.write("Waiting for launched jobs to finish .")
while (EDUtilsParallel.getNbRunning() > 0):
time.sleep(1)
sys.stderr.write(".")
sys.stderr.write("Done.\n")
def cleanUp(self, listMethods=[]):
"""
Final hook if you need to execute something after all processes finished (like killAllWorkers in SPD)
@param listMethods: allows to finish some things in the plugin.
@type listMethods: list of strings representing names of methods of the plugin to be called.
"""
self.waitForAllProcessToFinish()
edPlugin = EDParallelExecute.__edFactoryPlugin.loadPlugin(
self.__strPluginName)
for strOneMethod in listMethods:
try:
print "calling edPlugin.%s" % strOneMethod
exec "edPlugin.%s" % strOneMethod
except Exception:
print "error in processing " + strOneMethod
def testSaveModuleDictionaryToDisk(self):
edFactoryPlugin = EDFactoryPlugin()
edPluginTest = edFactoryPlugin.loadPlugin("EDPluginTestPluginFactory")
edFactoryPlugin.saveModuleDictionaryToDisk("testDictionary.xml")
def testLoadPlugin(self):
edFactoryPlugin = EDFactoryPlugin()
edPluginTest = edFactoryPlugin.loadPlugin("EDPluginTestPluginFactory")
EDAssert.equal("TestReturnValue", edPluginTest.getTestValue())
def testGetProjectName(self):
edFactoryPlugin = EDFactoryPlugin()
strProjectName = edFactoryPlugin.getProjectName("EDPluginTestPluginFactory")
EDAssert.equal("testProject", strProjectName)
def testGetProjectName(self):
edFactoryPlugin = EDFactoryPlugin()
strProjectName = edFactoryPlugin.getProjectName(
"EDPluginTestPluginFactory")
EDAssert.equal("testProject", strProjectName)
def testLoadPlugin(self):
edFactoryPlugin = EDFactoryPlugin()
edPluginTest = edFactoryPlugin.loadPlugin("EDPluginTestPluginFactory")
EDAssert.equal("TestReturnValue", edPluginTest.getTestValue())
def testSaveModuleDictionaryToDisk(self):
edFactoryPlugin = EDFactoryPlugin()
edPluginTest = edFactoryPlugin.loadPlugin("EDPluginTestPluginFactory")
edFactoryPlugin.saveModuleDictionaryToDisk("testDictionary.xml")
def testLoadModuleDictionaryFromDisk(self):
edFactoryPlugin = EDFactoryPlugin()
edFactoryPlugin.loadModuleDictionaryFromDisk("testDictionary.xml")
def testLoadModuleDictionaryFromDisk(self):
edFactoryPlugin = EDFactoryPlugin()
edFactoryPlugin.loadModuleDictionaryFromDisk("testDictionary.xml")
