def unitTestDetectNumberOfCPUs(self): """ test the execution of detectNumberOfCPUs """ iNbCPU = EDUtilsParallel.detectNumberOfCPUs() EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU) EDAssert.equal(types.IntType, type(iNbCPU), "Number of CPU is an integer") iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1 EDAssert.equal(1, iNbCPU, "Limit number of CPU") iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100 EDAssert.equal(100, iNbCPU, "Force number of CPU")
def unitTestDetectNumberOfCPUs(self): """ test the execution of detectNumberOfCPUs """ iNbCPU = EDUtilsParallel.detectNumberOfCPUs() EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU) EDAssert.equal(types.IntType, type(iNbCPU), "Number of CPU is an integer") iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1 EDAssert.equal(1, iNbCPU, "Limit number of CPU") iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100 EDAssert.equal(100, iNbCPU, "Force number of CPU")
def __init__(self, _strPluginName, _functXMLin, \ _functXMLout=None, _functXMLerr=None, \ _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None): """ This is the constructor of the edna plugin launcher. @param _strPluginName: the name of the ENDA plugin @type _strPluginName: python string @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. @type _functXMLin: python function @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout @type _functXMLOut: python function @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin @type _functXMLErr: python function @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected. @type _iNbThreads: python integer @param _fDelay: The delay in seconds between two directories analysis @type _fDelay: python float @param _bVerbose: Do you want the EDNA plugin execution to be verbose ? @type _bVerbose: boolean @param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ? @type _bDebug: boolean """ EDLogging.__init__(self) self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads) EDUtilsParallel.initializeNbThread(self.__iNbThreads) ################################################################################ # #We are not using the one from EDUtilsParallel to leave it to control the number of execPlugins. ################################################################################ self.__semaphoreNbThreads = Semaphore(self.__iNbThreads) self.__strPluginName = _strPluginName self.__functXMLin = _functXMLin self.__functXMLout = _functXMLout self.__functXMLerr = _functXMLerr self.__strCurrWorkDir = os.getcwd() self.__strTempDir = None self.__listInputPaths = [] self.__dictCurrentlyRunning = {} if _bVerbose is not None: if _bVerbose: self.setVerboseDebugOn() else: self.setVerboseOff() if _bDebug is not None: if _bDebug: self.setVerboseDebugOn() else: self.setVerboseDebugOff() self.__fDelay = _fDelay #default delay between two directory checks. self.__bQuit = False # To check if we should quit the application self.__bIsFirstExecute = True signal.signal(signal.SIGTERM, self.handleKill) signal.signal(signal.SIGINT, self.handleKill)
def __init__(self, _strPluginName, _functXMLin, \ _functXMLout=None, _functXMLerr=None, \ _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None): """ This is the constructor of the edna plugin launcher. @param _strPluginName: the name of the ENDA plugin @type _strPluginName: python string @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. @type _functXMLin: python function @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout @type _functXMLOut: python function @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin @type _functXMLErr: python function @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected. @type _iNbThreads: python integer @param _fDelay: The delay in seconds between two directories analysis @type _fDelay: python float @param _bVerbose: Do you want the EDNA plugin execution to be verbose ? @type _bVerbose: boolean @param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ? @type _bDebug: boolean """ EDLogging.__init__(self) self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads) EDUtilsParallel.initializeNbThread(self.__iNbThreads) ################################################################################ # #We are not using the one from EDUtilsParallel to leave it to control the number of execPlugins. ################################################################################ self.__semaphoreNbThreads = Semaphore(self.__iNbThreads) self.__strPluginName = _strPluginName self.__functXMLin = _functXMLin self.__functXMLout = _functXMLout self.__functXMLerr = _functXMLerr self.__strCurrWorkDir = os.getcwd() self.__strTempDir = None self.__listInputPaths = [] self.__dictCurrentlyRunning = {} if _bVerbose is not None: if _bVerbose: self.setVerboseDebugOn() else: self.setVerboseOff() if _bDebug is not None: if _bDebug: self.setVerboseDebugOn() else: self.setVerboseDebugOff() self.__fDelay = _fDelay #default delay between two directory checks. self.__bQuit = False # To check if we should quit the application self.__bIsFirstExecute = True signal.signal(signal.SIGTERM, self.handleKill) signal.signal(signal.SIGINT, self.handleKill)
def chooseWorker(self): """ selects the worker and sets self.worker """ # self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s" % self.getId()) if EDPluginSPDCorrectv10.__iNumberOfWorker == 0: EDPluginSPDCorrectv10.__iNumberOfWorker = EDUtilsParallel.detectNumberOfCPUs(self._iConfigNumberOfWorker) self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s nb of workers = %s lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__iNumberOfWorker, EDPluginSPDCorrectv10.__lockWorker._Semaphore__value)) # self.DEBUG("chooseWorker: \t EDPluginSPDCorrectv10.__lock.acquire(), currently: %i" % EDPluginSPDCorrectv10.__lock._Semaphore__value) EDPluginSPDCorrectv10.__lockWorker.acquire() for oneWorker in EDPluginSPDCorrectv10.__listOfWorker : self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Status of worker %i: %s" % (self.getId(), oneWorker.pid, oneWorker.getStatus())) if (oneWorker.getConfig() == self._SPDconfig) and (oneWorker.getStatus() in ["free", "uninitialized"]): self.worker = oneWorker if self.worker is None: if len(EDPluginSPDCorrectv10.__listOfWorker) < EDPluginSPDCorrectv10.__iNumberOfWorker : self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Initializing %i th worker" % (self.getId(), len(EDPluginSPDCorrectv10.__listOfWorker))) self.worker = SPDworker() EDPluginSPDCorrectv10.__listOfWorker.append(self.worker) self.worker.setExecutable(self.getExecutable()) self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker) self.worker.setLogFilename(os.path.join(self.getSPDCommonDirectory(), "worker-%02i.log" % self.workerID)) self.worker.initialize(self._SPDconfig) self.worker.setTimeOut(self.getTimeOut()) while self.worker is None: self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: No workers still to initialize" % self.getId()) bConfigOK = False for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker) : # self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), oneWorker.getConfig())) # self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), self._SPDconfig)) #bug1 those string must specifically be the same ... change line 248 in spdCake1.5 if (oneWorker.getConfig() == self._SPDconfig): bConfigOK = True if (oneWorker.getStatus() in ["free", "uninitialized"]): self.worker = oneWorker self.workerID = idx if bConfigOK == False: for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker[:]) : # self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: worker %i status %s " % (self.getId(), idx, oneWorker.status)) if (oneWorker.getStatus() in ["free", "uninitialized"]): #Bug2 why doest this work ??? # self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Resetting worker %i status %s " % (self.getId(), idx, oneWorker.status)) oneWorker.initialize(self._SPDconfig) self.worker = oneWorker self.workerID = idx EDPluginSPDCorrectv10.__listOfWorker[idx] = (self.worker) self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Left loop worker %s status %s " % (self.getId(), self.workerID, oneWorker)) time.sleep(0.1) if self.workerID is None: self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker) self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Release lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__lockWorker._Semaphore__value)) EDPluginSPDCorrectv10.__lockWorker.release()
def __init__(self, cl, name): EDLogging.__init__(self) PyTango.Device_4Impl.__init__(self, cl, name) self.init_device() if isinstance(iNbCpu, int): self.screen("Initializing tangoDS with max %i jobs in parallel." % iNbCpu) self.__semaphoreNbThreads = threading.Semaphore(iNbCpu) else: self.__semaphoreNbThreads = threading.Semaphore(EDUtilsParallel.detectNumberOfCPUs()) self.jobQueue = Queue() self.processingSem = threading.Semaphore() self.statLock = threading.Lock() self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first" self.lastFailure = "No job Failed (yet)" self.lastSuccess = "No job succeeded (yet)"
def __init__(self, cl, name): EDLogging.__init__(self) PyTango.Device_4Impl.__init__(self, cl, name) self.init_device() if isinstance(iNbCpu, int): self.screen("Initializing tangoDS with max %i jobs in parallel." % iNbCpu) self.__semaphoreNbThreads = threading.Semaphore(iNbCpu) else: self.__semaphoreNbThreads = threading.Semaphore( EDUtilsParallel.detectNumberOfCPUs()) self.jobQueue = Queue() self.processingSem = threading.Semaphore() self.statLock = threading.Lock() self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first" self.lastFailure = "No job Failed (yet)" self.lastSuccess = "No job succeeded (yet)"
def __init__(self, strPluginName, iNbCpu=None): EDLogging.__init__(self) self.pluginName = strPluginName self.startTime = time.time() try: self.iNbCpu = int(iNbCpu) except: self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs() self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu) self.__semaphoreNbThreads = Semaphore(self.iNbCpu) EDUtilsParallel.initializeNbThread(self.iNbCpu) self.jobQueue = Queue() self.processingSem = Semaphore() self.statLock = Semaphore() self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first" self.lastFailure = "No job Failed (yet)" self.lastSuccess = "No job succeeded (yet)"
def process(self, _edPlugin=None): """ Executes the action cluster. This method will return only when all actions have finished. """ self.DEBUG("EDActionCluster.process") if (not self.__iNoThreads): self.__iNoThreads = EDUtilsParallel.detectNumberOfCPUs() self.__semaphoreActionCluster = threading.Semaphore(self.__iNoThreads) for edAction in self.__lActions: edAction.connectSUCCESS(self.__semaphoreRelease) edAction.connectFAILURE(self.__setActionClusterFailure) edAction.connectFAILURE(self.__semaphoreRelease) self.DEBUG("EDActionCluster.process : Starting action %s" % edAction.getClassName()) self.__semaphoreActionCluster.acquire() edAction.execute() # Wait for all threads to finish for edAction in self.__lActions: edAction.join()
def process(self, _edPlugin=None): """ Executes the action cluster. This method will return only when all actions have finished. """ self.DEBUG("EDActionCluster.process") if (not self.__iNoThreads): self.__iNoThreads = EDUtilsParallel.detectNumberOfCPUs() self.__semaphoreActionCluster = threading.Semaphore(self.__iNoThreads) for edAction in self.__lActions: edAction.connectSUCCESS(self.__semaphoreRelease) edAction.connectFAILURE(self.__setActionClusterFailure) edAction.connectFAILURE(self.__semaphoreRelease) self.DEBUG("EDActionCluster.process : Starting action %s" % edAction.getClassName()) self.__semaphoreActionCluster.acquire() edAction.execute() # Wait for all threads to finish for edAction in self.__lActions: edAction.join()
def __init__(self, cl, name): EDLogging.__init__(self) PyTango.Device_4Impl.__init__(self, cl, name) self.init_device() if isinstance(iNbCpu, int): self.screen("Initializing tangoDS with max %i jobs in parallel." % iNbCpu) self.__semaphoreNbThreads = threading.Semaphore(iNbCpu) else: self.__semaphoreNbThreads = threading.Semaphore(EDUtilsParallel.detectNumberOfCPUs()) self.quit = False self.jobQueue = Queue() # queue containing jobs to process self.eventQueue = Queue() # queue containing finished jobs self.statLock = threading.Lock() self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first" self.lastFailure = "No job Failed (yet)" self.lastSuccess = "No job succeeded (yet)" self.processingThread = threading.Thread(target=self.startProcessing) self.processingThread.start() self.finishingThread = threading.Thread(target=self.process_event) self.finishingThread.start()
installHDF5.downloadLibrary() installHDF5.unZipArchive() pthreadPath = installHDF5.searchCLib("libpthread.so") EDVerbose.DEBUG("Libpthread found in %s" % pthreadPath) if pthreadPath is None: try: installHDF5.configure("--prefix=%s" % (installHDF5.getDestinationDirectory())) except: EDVerbose.ERROR("Error in the configure step, no pthread") else: try: installHDF5.configure("--prefix=%s --enable-threadsafe --with-pthread=%s" % (installHDF5.getDestinationDirectory(), pthreadPath)) except: EDVerbose.ERROR("Error in the configure step, with pthread") try: installHDF5.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs()) except: EDVerbose.ERROR("Error in the 'make' step") try: installHDF5.make("install") except: EDVerbose.ERROR("Error in the 'make install' step") hdfPath = installHDF5.getDestinationDirectory() installHDF5.cleanSources() else: hdfPath = os.path.dirname(hdfPath) EDVerbose.DEBUG("Building H5Py with HDF5 library from %s " % (hdfPath)) install = EDUtilsLibraryInstaller(installDir, h5pyLibrary) install.checkPythonVersion() install.dependency("numpy", "20090405-Numpy-1.3")
makeFile.write("LOADER = %s %s" % (fortranCompiler, os.linesep)) # elif oneline.startswith("ARCHFLAGS"): # makeFile.write("ARCHFLAGS = -shared -o %s" % (os.linesep)) # elif oneline.startswith("ARCH"): # makeFile.write("ARCH = %s %s" % (fortranCompiler, os.linesep)) # elif oneline.startswith("RANLIB"): # makeFile.write("RANLIB = ls %s" % (os.linesep)) elif oneline.startswith("BLASLIB"): makeFile.write("BLASLIB = %s %s" % (libblas, os.linesep)) else: makeFile.write(oneline) makeFile.close() try: installBlas.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs()) except Exception: EDVerbose.ERROR("Error for BLAS in the 'make' step") if not os.path.isdir( os.path.join(installBlas.getDestinationDirectory(), "lib")): os.makedirs( os.path.join(installBlas.getDestinationDirectory(), "lib")) blasPath = os.path.join(installBlas.getDestinationDirectory(), "lib") shutil.copyfile( os.path.join(installBlas.getSourceDirectory(), libblas), os.path.join(blasPath, libblas)) ################################################################################ # END of Blas / Start of Lapack ################################################################################ if lapackPath is None:
elif oneline.startswith("LOADER"): makeFile.write("LOADER = %s %s" % (fortranCompiler, os.linesep)) # elif oneline.startswith("ARCHFLAGS"): # makeFile.write("ARCHFLAGS = -shared -o %s" % (os.linesep)) # elif oneline.startswith("ARCH"): # makeFile.write("ARCH = %s %s" % (fortranCompiler, os.linesep)) # elif oneline.startswith("RANLIB"): # makeFile.write("RANLIB = ls %s" % (os.linesep)) elif oneline.startswith("BLASLIB"): makeFile.write("BLASLIB = %s %s" % (libblas, os.linesep)) else: makeFile.write(oneline) makeFile.close() try: installBlas.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs()) except Exception: EDVerbose.ERROR("Error for BLAS in the 'make' step") if not os.path.isdir(os.path.join(installBlas.getDestinationDirectory(), "lib")): os.makedirs(os.path.join(installBlas.getDestinationDirectory(), "lib")) blasPath = os.path.join(installBlas.getDestinationDirectory(), "lib") shutil.copyfile(os.path.join(installBlas.getSourceDirectory(), libblas), os.path.join(blasPath, libblas)) ################################################################################ # END of Blas / Start of Lapack ################################################################################ if lapackPath is None: EDVerbose.screen("Checking for Lapack %s : not found, so I have to compile it myself" % liblapack) else: EDVerbose.screen("Checking for Lapack %s : Found on %s, but I don't trust it because Blas was missing" % (liblapack, lapackPath)) liblapack = os.path.splitext(liblapack)[0] + ".a"
EDVerbose.DEBUG("Libpthread found in %s" % pthreadPath) if pthreadPath is None: try: installHDF5.configure("--prefix=%s" % (installHDF5.getDestinationDirectory())) except Exception: EDVerbose.ERROR("Error in the configure step, no pthread") else: try: installHDF5.configure( "--prefix=%s --enable-threadsafe --with-pthread=%s" % (installHDF5.getDestinationDirectory(), pthreadPath)) except Exception: EDVerbose.ERROR("Error in the configure step, with pthread") try: installHDF5.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs()) except Exception: EDVerbose.ERROR("Error in the 'make' step") try: installHDF5.make("install") except Exception: EDVerbose.ERROR("Error in the 'make install' step") hdfPath = installHDF5.getDestinationDirectory() installHDF5.cleanSources() else: hdfPath = os.path.dirname(hdfPath) EDVerbose.DEBUG("Building H5Py with HDF5 library from %s " % (hdfPath)) install = EDUtilsLibraryInstaller(installDir, h5pyLibrary) install.checkPythonVersion() install.dependency("numpy", "20090405-Numpy-1.3")