def unitTestDetectNumberOfCPUs(self):
"""
test the execution of detectNumberOfCPUs
"""
iNbCPU = EDUtilsParallel.detectNumberOfCPUs()
EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU)
EDAssert.equal(types.IntType, type(iNbCPU), "Number of CPU is an integer")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1
EDAssert.equal(1, iNbCPU, "Limit number of CPU")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100
EDAssert.equal(100, iNbCPU, "Force number of CPU")
def unitTestDetectNumberOfCPUs(self):
"""
test the execution of detectNumberOfCPUs
"""
iNbCPU = EDUtilsParallel.detectNumberOfCPUs()
EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU)
EDAssert.equal(types.IntType, type(iNbCPU),
"Number of CPU is an integer")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1
EDAssert.equal(1, iNbCPU, "Limit number of CPU")
iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100
EDAssert.equal(100, iNbCPU, "Force number of CPU")
def __init__(self, _strPluginName, _functXMLin, \
_functXMLout=None, _functXMLerr=None, \
_iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
"""
This is the constructor of the edna plugin launcher.
@param _strPluginName: the name of the ENDA plugin
@type _strPluginName: python string
@param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin.
@type _functXMLin: python function
@param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
@type _functXMLOut: python function
@param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
@type _functXMLErr: python function
@param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected.
@type _iNbThreads: python integer
@param _fDelay: The delay in seconds between two directories analysis
@type _fDelay: python float
@param _bVerbose: Do you want the EDNA plugin execution to be verbose ?
@type _bVerbose: boolean
@param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ?
@type _bDebug: boolean
"""
EDLogging.__init__(self)
self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of execPlugins.
################################################################################
self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
self.__strPluginName = _strPluginName
self.__functXMLin = _functXMLin
self.__functXMLout = _functXMLout
self.__functXMLerr = _functXMLerr
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = None
self.__listInputPaths = []
self.__dictCurrentlyRunning = {}
if _bVerbose is not None:
if _bVerbose:
self.setVerboseDebugOn()
else:
self.setVerboseOff()
if _bDebug is not None:
if _bDebug:
self.setVerboseDebugOn()
else:
self.setVerboseDebugOff()
self.__fDelay = _fDelay #default delay between two directory checks.
self.__bQuit = False # To check if we should quit the application
self.__bIsFirstExecute = True
signal.signal(signal.SIGTERM, self.handleKill)
signal.signal(signal.SIGINT, self.handleKill)
def __init__(self, _strPluginName, _functXMLin, \
_functXMLout=None, _functXMLerr=None, \
_iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
"""
This is the constructor of the edna plugin launcher.
@param _strPluginName: the name of the ENDA plugin
@type _strPluginName: python string
@param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin.
@type _functXMLin: python function
@param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
@type _functXMLOut: python function
@param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
@type _functXMLErr: python function
@param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected.
@type _iNbThreads: python integer
@param _fDelay: The delay in seconds between two directories analysis
@type _fDelay: python float
@param _bVerbose: Do you want the EDNA plugin execution to be verbose ?
@type _bVerbose: boolean
@param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ?
@type _bDebug: boolean
"""
EDLogging.__init__(self)
self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of execPlugins.
################################################################################
self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
self.__strPluginName = _strPluginName
self.__functXMLin = _functXMLin
self.__functXMLout = _functXMLout
self.__functXMLerr = _functXMLerr
self.__strCurrWorkDir = os.getcwd()
self.__strTempDir = None
self.__listInputPaths = []
self.__dictCurrentlyRunning = {}
if _bVerbose is not None:
if _bVerbose:
self.setVerboseDebugOn()
else:
self.setVerboseOff()
if _bDebug is not None:
if _bDebug:
self.setVerboseDebugOn()
else:
self.setVerboseDebugOff()
self.__fDelay = _fDelay #default delay between two directory checks.
self.__bQuit = False # To check if we should quit the application
self.__bIsFirstExecute = True
signal.signal(signal.SIGTERM, self.handleKill)
signal.signal(signal.SIGINT, self.handleKill)
def chooseWorker(self):
"""
selects the worker and sets self.worker
"""
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s" % self.getId())
if EDPluginSPDCorrectv10.__iNumberOfWorker == 0:
EDPluginSPDCorrectv10.__iNumberOfWorker = EDUtilsParallel.detectNumberOfCPUs(self._iConfigNumberOfWorker)
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s nb of workers = %s lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__iNumberOfWorker, EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
# self.DEBUG("chooseWorker: \t EDPluginSPDCorrectv10.__lock.acquire(), currently: %i" % EDPluginSPDCorrectv10.__lock._Semaphore__value)
EDPluginSPDCorrectv10.__lockWorker.acquire()
for oneWorker in EDPluginSPDCorrectv10.__listOfWorker :
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Status of worker %i: %s" % (self.getId(), oneWorker.pid, oneWorker.getStatus()))
if (oneWorker.getConfig() == self._SPDconfig) and (oneWorker.getStatus() in ["free", "uninitialized"]):
self.worker = oneWorker
if self.worker is None:
if len(EDPluginSPDCorrectv10.__listOfWorker) < EDPluginSPDCorrectv10.__iNumberOfWorker :
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Initializing %i th worker" % (self.getId(), len(EDPluginSPDCorrectv10.__listOfWorker)))
self.worker = SPDworker()
EDPluginSPDCorrectv10.__listOfWorker.append(self.worker)
self.worker.setExecutable(self.getExecutable())
self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
self.worker.setLogFilename(os.path.join(self.getSPDCommonDirectory(), "worker-%02i.log" % self.workerID))
self.worker.initialize(self._SPDconfig)
self.worker.setTimeOut(self.getTimeOut())
while self.worker is None:
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: No workers still to initialize" % self.getId())
bConfigOK = False
for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker) :
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), oneWorker.getConfig()))
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), self._SPDconfig))
#bug1 those string must specifically be the same ... change line 248 in spdCake1.5
if (oneWorker.getConfig() == self._SPDconfig):
bConfigOK = True
if (oneWorker.getStatus() in ["free", "uninitialized"]):
self.worker = oneWorker
self.workerID = idx
if bConfigOK == False:
for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker[:]) :
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: worker %i status %s " % (self.getId(), idx, oneWorker.status))
if (oneWorker.getStatus() in ["free", "uninitialized"]):
#Bug2 why doest this work ???
# self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Resetting worker %i status %s " % (self.getId(), idx, oneWorker.status))
oneWorker.initialize(self._SPDconfig)
self.worker = oneWorker
self.workerID = idx
EDPluginSPDCorrectv10.__listOfWorker[idx] = (self.worker)
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Left loop worker %s status %s " % (self.getId(), self.workerID, oneWorker))
time.sleep(0.1)
if self.workerID is None:
self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Release lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
EDPluginSPDCorrectv10.__lockWorker.release()
def __init__(self, cl, name):
EDLogging.__init__(self)
PyTango.Device_4Impl.__init__(self, cl, name)
self.init_device()
if isinstance(iNbCpu, int):
self.screen("Initializing tangoDS with max %i jobs in parallel." % iNbCpu)
self.__semaphoreNbThreads = threading.Semaphore(iNbCpu)
else:
self.__semaphoreNbThreads = threading.Semaphore(EDUtilsParallel.detectNumberOfCPUs())
self.jobQueue = Queue()
self.processingSem = threading.Semaphore()
self.statLock = threading.Lock()
self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
self.lastFailure = "No job Failed (yet)"
self.lastSuccess = "No job succeeded (yet)"
def __init__(self, cl, name):
EDLogging.__init__(self)
PyTango.Device_4Impl.__init__(self, cl, name)
self.init_device()
if isinstance(iNbCpu, int):
self.screen("Initializing tangoDS with max %i jobs in parallel." %
iNbCpu)
self.__semaphoreNbThreads = threading.Semaphore(iNbCpu)
else:
self.__semaphoreNbThreads = threading.Semaphore(
EDUtilsParallel.detectNumberOfCPUs())
self.jobQueue = Queue()
self.processingSem = threading.Semaphore()
self.statLock = threading.Lock()
self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
self.lastFailure = "No job Failed (yet)"
self.lastSuccess = "No job succeeded (yet)"
def __init__(self, strPluginName, iNbCpu=None):
EDLogging.__init__(self)
self.pluginName = strPluginName
self.startTime = time.time()
try:
self.iNbCpu = int(iNbCpu)
except:
self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs()
self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu)
self.__semaphoreNbThreads = Semaphore(self.iNbCpu)
EDUtilsParallel.initializeNbThread(self.iNbCpu)
self.jobQueue = Queue()
self.processingSem = Semaphore()
self.statLock = Semaphore()
self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
self.lastFailure = "No job Failed (yet)"
self.lastSuccess = "No job succeeded (yet)"
def process(self, _edPlugin=None):
"""
Executes the action cluster. This method will return only when
all actions have finished.
"""
self.DEBUG("EDActionCluster.process")
if (not self.__iNoThreads):
self.__iNoThreads = EDUtilsParallel.detectNumberOfCPUs()
self.__semaphoreActionCluster = threading.Semaphore(self.__iNoThreads)
for edAction in self.__lActions:
edAction.connectSUCCESS(self.__semaphoreRelease)
edAction.connectFAILURE(self.__setActionClusterFailure)
edAction.connectFAILURE(self.__semaphoreRelease)
self.DEBUG("EDActionCluster.process : Starting action %s" % edAction.getClassName())
self.__semaphoreActionCluster.acquire()
edAction.execute()
# Wait for all threads to finish
for edAction in self.__lActions:
edAction.join()
def process(self, _edPlugin=None):
"""
Executes the action cluster. This method will return only when
all actions have finished.
"""
self.DEBUG("EDActionCluster.process")
if (not self.__iNoThreads):
self.__iNoThreads = EDUtilsParallel.detectNumberOfCPUs()
self.__semaphoreActionCluster = threading.Semaphore(self.__iNoThreads)
for edAction in self.__lActions:
edAction.connectSUCCESS(self.__semaphoreRelease)
edAction.connectFAILURE(self.__setActionClusterFailure)
edAction.connectFAILURE(self.__semaphoreRelease)
self.DEBUG("EDActionCluster.process : Starting action %s" %
edAction.getClassName())
self.__semaphoreActionCluster.acquire()
edAction.execute()
# Wait for all threads to finish
for edAction in self.__lActions:
edAction.join()
def __init__(self, cl, name):
EDLogging.__init__(self)
PyTango.Device_4Impl.__init__(self, cl, name)
self.init_device()
if isinstance(iNbCpu, int):
self.screen("Initializing tangoDS with max %i jobs in parallel." % iNbCpu)
self.__semaphoreNbThreads = threading.Semaphore(iNbCpu)
else:
self.__semaphoreNbThreads = threading.Semaphore(EDUtilsParallel.detectNumberOfCPUs())
self.quit = False
self.jobQueue = Queue() # queue containing jobs to process
self.eventQueue = Queue() # queue containing finished jobs
self.statLock = threading.Lock()
self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
self.lastFailure = "No job Failed (yet)"
self.lastSuccess = "No job succeeded (yet)"
self.processingThread = threading.Thread(target=self.startProcessing)
self.processingThread.start()
self.finishingThread = threading.Thread(target=self.process_event)
self.finishingThread.start()
installHDF5.downloadLibrary()
installHDF5.unZipArchive()
pthreadPath = installHDF5.searchCLib("libpthread.so")
EDVerbose.DEBUG("Libpthread found in %s" % pthreadPath)
if pthreadPath is None:
try:
installHDF5.configure("--prefix=%s" % (installHDF5.getDestinationDirectory()))
except:
EDVerbose.ERROR("Error in the configure step, no pthread")
else:
try:
installHDF5.configure("--prefix=%s --enable-threadsafe --with-pthread=%s" % (installHDF5.getDestinationDirectory(), pthreadPath))
except:
EDVerbose.ERROR("Error in the configure step, with pthread")
try:
installHDF5.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
except:
EDVerbose.ERROR("Error in the 'make' step")
try:
installHDF5.make("install")
except:
EDVerbose.ERROR("Error in the 'make install' step")
hdfPath = installHDF5.getDestinationDirectory()
installHDF5.cleanSources()
else:
hdfPath = os.path.dirname(hdfPath)
EDVerbose.DEBUG("Building H5Py with HDF5 library from %s " % (hdfPath))
install = EDUtilsLibraryInstaller(installDir, h5pyLibrary)
install.checkPythonVersion()
install.dependency("numpy", "20090405-Numpy-1.3")
makeFile.write("LOADER = %s %s" %
(fortranCompiler, os.linesep))
# elif oneline.startswith("ARCHFLAGS"):
# makeFile.write("ARCHFLAGS = -shared -o %s" % (os.linesep))
# elif oneline.startswith("ARCH"):
# makeFile.write("ARCH = %s %s" % (fortranCompiler, os.linesep))
# elif oneline.startswith("RANLIB"):
# makeFile.write("RANLIB = ls %s" % (os.linesep))
elif oneline.startswith("BLASLIB"):
makeFile.write("BLASLIB = %s %s" % (libblas, os.linesep))
else:
makeFile.write(oneline)
makeFile.close()
try:
installBlas.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
except Exception:
EDVerbose.ERROR("Error for BLAS in the 'make' step")
if not os.path.isdir(
os.path.join(installBlas.getDestinationDirectory(), "lib")):
os.makedirs(
os.path.join(installBlas.getDestinationDirectory(), "lib"))
blasPath = os.path.join(installBlas.getDestinationDirectory(), "lib")
shutil.copyfile(
os.path.join(installBlas.getSourceDirectory(), libblas),
os.path.join(blasPath, libblas))
################################################################################
# END of Blas / Start of Lapack
################################################################################
if lapackPath is None:
elif oneline.startswith("LOADER"):
makeFile.write("LOADER = %s %s" % (fortranCompiler, os.linesep))
# elif oneline.startswith("ARCHFLAGS"):
# makeFile.write("ARCHFLAGS = -shared -o %s" % (os.linesep))
# elif oneline.startswith("ARCH"):
# makeFile.write("ARCH = %s %s" % (fortranCompiler, os.linesep))
# elif oneline.startswith("RANLIB"):
# makeFile.write("RANLIB = ls %s" % (os.linesep))
elif oneline.startswith("BLASLIB"):
makeFile.write("BLASLIB = %s %s" % (libblas, os.linesep))
else:
makeFile.write(oneline)
makeFile.close()
try:
installBlas.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
except Exception:
EDVerbose.ERROR("Error for BLAS in the 'make' step")
if not os.path.isdir(os.path.join(installBlas.getDestinationDirectory(), "lib")):
os.makedirs(os.path.join(installBlas.getDestinationDirectory(), "lib"))
blasPath = os.path.join(installBlas.getDestinationDirectory(), "lib")
shutil.copyfile(os.path.join(installBlas.getSourceDirectory(), libblas), os.path.join(blasPath, libblas))
################################################################################
# END of Blas / Start of Lapack
################################################################################
if lapackPath is None:
EDVerbose.screen("Checking for Lapack %s : not found, so I have to compile it myself" % liblapack)
else:
EDVerbose.screen("Checking for Lapack %s : Found on %s, but I don't trust it because Blas was missing" % (liblapack, lapackPath))
liblapack = os.path.splitext(liblapack)[0] + ".a"
EDVerbose.DEBUG("Libpthread found in %s" % pthreadPath)
if pthreadPath is None:
try:
installHDF5.configure("--prefix=%s" %
(installHDF5.getDestinationDirectory()))
except Exception:
EDVerbose.ERROR("Error in the configure step, no pthread")
else:
try:
installHDF5.configure(
"--prefix=%s --enable-threadsafe --with-pthread=%s" %
(installHDF5.getDestinationDirectory(), pthreadPath))
except Exception:
EDVerbose.ERROR("Error in the configure step, with pthread")
try:
installHDF5.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
except Exception:
EDVerbose.ERROR("Error in the 'make' step")
try:
installHDF5.make("install")
except Exception:
EDVerbose.ERROR("Error in the 'make install' step")
hdfPath = installHDF5.getDestinationDirectory()
installHDF5.cleanSources()
else:
hdfPath = os.path.dirname(hdfPath)
EDVerbose.DEBUG("Building H5Py with HDF5 library from %s " % (hdfPath))
install = EDUtilsLibraryInstaller(installDir, h5pyLibrary)
install.checkPythonVersion()
install.dependency("numpy", "20090405-Numpy-1.3")
