Ejemplo n.º 1
0
    def __init__(self, _strPluginName, _functXMLin, \
                  _functXMLout=None, _functXMLerr=None, \
                  _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
        """
        This is the constructor of the edna plugin launcher.
        
        @param _strPluginName: the name of the ENDA plugin
        @type  _strPluginName: python string
        
        @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. 
        @type  _functXMLin: python function
        
        @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
        @type  _functXMLOut: python function
         
        @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
        @type  _functXMLErr: python function 
        
        @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores  will be auto-detected. 
        @type  _iNbThreads: python integer
        
        @param _fDelay: The delay in seconds between two directories analysis 
        @type  _fDelay: python float
        
        @param _bVerbose:  Do you want the EDNA plugin execution to be verbose ?
        @type  _bVerbose: boolean

        @param _bDebug:  Do you want EDNA plugin execution debug output (OBS! very verbose) ?
        @type  _bDebug: boolean
        """
        EDLogging.__init__(self)
        self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
        EDUtilsParallel.initializeNbThread(self.__iNbThreads)
        ################################################################################
        # #We are not using the one from EDUtilsParallel to leave it to control the number of  execPlugins.
        ################################################################################
        self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
        self.__strPluginName = _strPluginName
        self.__functXMLin = _functXMLin
        self.__functXMLout = _functXMLout
        self.__functXMLerr = _functXMLerr
        self.__strCurrWorkDir = os.getcwd()
        self.__strTempDir = None
        self.__listInputPaths = []
        self.__dictCurrentlyRunning = {}
        if _bVerbose is not None:
            if _bVerbose:
                self.setVerboseDebugOn()
            else:
                self.setVerboseOff()
        if _bDebug is not None:
            if _bDebug:
                self.setVerboseDebugOn()
            else:
                self.setVerboseDebugOff()
        self.__fDelay = _fDelay  #default delay between two directory checks.
        self.__bQuit = False  # To check if we should quit the application
        self.__bIsFirstExecute = True
        signal.signal(signal.SIGTERM, self.handleKill)
        signal.signal(signal.SIGINT, self.handleKill)
Ejemplo n.º 2
0
    def __init__(self, _strPluginName, _functXMLin, \
                  _functXMLout=None, _functXMLerr=None, \
                  _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
        """
        This is the constructor of the edna plugin launcher.
        
        @param _strPluginName: the name of the ENDA plugin
        @type  _strPluginName: python string
        
        @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. 
        @type  _functXMLin: python function
        
        @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
        @type  _functXMLOut: python function
         
        @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
        @type  _functXMLErr: python function 
        
        @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores  will be auto-detected. 
        @type  _iNbThreads: python integer
        
        @param _fDelay: The delay in seconds between two directories analysis 
        @type  _fDelay: python float
        
        @param _bVerbose:  Do you want the EDNA plugin execution to be verbose ?
        @type  _bVerbose: boolean

        @param _bDebug:  Do you want EDNA plugin execution debug output (OBS! very verbose) ?
        @type  _bDebug: boolean
        """
        EDLogging.__init__(self)
        self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
        EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of  execPlugins.
################################################################################
        self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
        self.__strPluginName = _strPluginName
        self.__functXMLin = _functXMLin
        self.__functXMLout = _functXMLout
        self.__functXMLerr = _functXMLerr
        self.__strCurrWorkDir = os.getcwd()
        self.__strTempDir = None
        self.__listInputPaths = []
        self.__dictCurrentlyRunning = {}
        if _bVerbose is not None:
            if _bVerbose:
                self.setVerboseDebugOn()
            else:
                self.setVerboseOff()
        if _bDebug is not None:
            if _bDebug:
                self.setVerboseDebugOn()
            else:
                self.setVerboseDebugOff()
        self.__fDelay = _fDelay #default delay between two directory checks.
        self.__bQuit = False    # To check if we should quit the application
        self.__bIsFirstExecute = True
        signal.signal(signal.SIGTERM, self.handleKill)
        signal.signal(signal.SIGINT, self.handleKill)
    def preProcess(self):
        """
        PreProcess of the execution test: download a set of images  from http://www.edna-site.org
        and remove any existing output file 
        """
        EDTestCasePluginExecute.preProcess(self)
        self.loadTestImage([ "bioSaxsMask.edf", "bioSaxsNormalized.edf", "bioSaxsIntegratedv1_3.dat", "bioSaxsCorrected.edf"])
        strExpectedOutput = self.readAndParseFile (self.getReferenceDataOutputFile())
        EDVerbose.DEBUG("strExpectedOutput:" + strExpectedOutput)
        xsDataResultReference = XSDataResultBioSaxsAzimutIntv1_0.parseString(strExpectedOutput)
        self.integratedCurve = xsDataResultReference.getIntegratedCurve().getPath().value
        EDVerbose.DEBUG("Output file is %s" % self.integratedCurve)
        if not os.path.isdir(os.path.dirname(self.integratedCurve)):
            os.makedirs(os.path.dirname(self.integratedCurve))
        if os.path.isfile(self.integratedCurve):
            EDVerbose.DEBUG(" Output Integrated Curve file exists %s, I will remove it" % self.integratedCurve)
            os.remove(self.integratedCurve)

        EDUtilsParallel.initializeNbThread()
#        self.correctedImage = xsDataResultReference.getCorrectedImage().getPath().value
#        EDVerbose.DEBUG("Output Corrected Image file is %s" % self.correctedImage)
#        if not os.path.isdir(os.path.dirname(self.correctedImage)):
#            os.makedirs(os.path.dirname(self.correctedImage))
#        if os.path.isfile(self.correctedImage):
#            EDVerbose.DEBUG(" Output Corrected Image file exists %s, I will remove it" % self.correctedImage)
#            os.remove(self.correctedImage)
        if not pyFAI.version.startswith("0.7"):
            EDVerbose.ERROR('pyFAI is not the right version, tested only with 0.7.2, here running version %s' % pyFAI.version)
Ejemplo n.º 4
0
    def start(self, _strXmlInput):
        """
        Launch EDNA with the given XML stream
        @param _strXmlInput:  XML to be passed to the plugin
        @type  _strXmlInput: python string representing the XML data structure
        """
        if (_strXmlInput is None) or (_strXmlInput == "") :
            return
        #This is a trick to work-around bug #463:
        #  Run the fist thread alone (delay the second, third, ...)
        #    semaphore._Semaphore__value is the current value of the value, unfortunatly it is a protected value without getter 
        #    I need the value of the semaphore to guess if the current call is the first or not.
        #    Nota semaphore are decreased in value from self.__iNbThreads to 0. When Zero, the semaphore is blocking.
        # Them all other, limited by the semaphore.

        if self.__bIsFirstExecute:
            sys.stdout.write("Waiting for first thread to initialize ....")
            while EDUtilsParallel.getNbRunning() > 0:
                time.sleep(0.5)
                sys.stdout.write(".")
        EDUtilsParallel.semaphoreNbThreadsAcquire()

        edPlugin = EDParallelExecute.__edFactoryPlugin.loadPlugin(self.__strPluginName)

        if (edPlugin is not None):

            edPlugin.setDataInput(_strXmlInput)
            edPlugin.connectSUCCESS(self.successPluginExecution)
            edPlugin.connectFAILURE(self.failurePluginExecution)
            edPlugin.execute()
            #edPlugin.executeSynchronous()
        else:
            EDUtilsParallel.semaphoreNbThreadsRelease()
            self.__bIsFirstExecute = False
            EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName)
Ejemplo n.º 5
0
 def preProcess(self):
     """
     PreProcess of the execution test: download a set of images  from http://www.edna-site.org
     and remove any existing output file 
     """
     EDTestCasePluginExecute.preProcess(self)
     self.loadTestImage([
         "bioSaxsRaw.edf", "bioSaxsMask.edf", "bioSaxsCorrected.edf",
         "bioSaxsProcessNormalized.edf", "bioSaxsProcessIntegrated.edf",
         "bioSaxsProcessIntegrated1_2.dat"
     ])
     strExpectedOutput = self.readAndParseFile(
         self.getReferenceDataOutputFile())
     EDVerbose.DEBUG("strExpectedOutput:" + strExpectedOutput)
     xsDataResultReference = XSDataResultBioSaxsProcessOneFilev1_0.parseString(
         strExpectedOutput)
     self.refNormImg = xsDataResultReference.normalizedImage.path.value
     self.refIntCrv = xsDataResultReference.integratedCurve.path.value
     if not os.path.isdir(os.path.dirname(self.refNormImg)):
         os.makedirs(os.path.dirname(self.refNormImg))
     if os.path.isfile(self.refNormImg):
         os.remove(self.refNormImg)
     if os.path.isfile(self.refIntCrv):
         os.remove(self.refIntCrv)
     EDUtilsParallel.initializeNbThread()
    def preProcess(self):
        """
        PreProcess of the execution test: download a set of images  from http://www.edna-site.org
        and remove any existing output file 
        """
        EDTestCasePluginExecute.preProcess(self)
        self.loadTestImage([
            "bioSaxsMask.edf", "bioSaxsNormalized.edf",
            "bioSaxsIntegratedv1_3.dat", "bioSaxsCorrected.edf"
        ])
        strExpectedOutput = self.readAndParseFile(
            self.getReferenceDataOutputFile())
        EDVerbose.DEBUG("strExpectedOutput:" + strExpectedOutput)
        xsDataResultReference = XSDataResultBioSaxsAzimutIntv1_0.parseString(
            strExpectedOutput)
        self.integratedCurve = xsDataResultReference.getIntegratedCurve(
        ).getPath().value
        EDVerbose.DEBUG("Output file is %s" % self.integratedCurve)
        if not os.path.isdir(os.path.dirname(self.integratedCurve)):
            os.makedirs(os.path.dirname(self.integratedCurve))
        if os.path.isfile(self.integratedCurve):
            EDVerbose.DEBUG(
                " Output Integrated Curve file exists %s, I will remove it" %
                self.integratedCurve)
            os.remove(self.integratedCurve)

        EDUtilsParallel.initializeNbThread()
Ejemplo n.º 7
0
 def preProcess(self, _edObject=None):
     """
     preProcess of the plugin:
     Ensure a CPU resource is available for the processing by acquiring a semaphore
     """
     self.DEBUG("Acquire semaphore nbCPU by plugin %s" % self.getPluginName())
     EDUtilsParallel.semaphoreNbThreadsAcquire()
     EDPlugin.preProcess(self, _edObject)
Ejemplo n.º 8
0
 def finallyProcess(self, _edObject=None):
     """
     after processing of the plugin:
     Release a CPU resource by releasing the semaphore
     """
     EDVerbose.DEBUG("Release semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
     EDUtilsParallel.semaphoreNbThreadsRelease()
     EDPlugin.finallyProcess(self, _edObject)
Ejemplo n.º 9
0
 def preProcess(self, _edObject=None):
     """
     preProcess of the plugin:
     Ensure a CPU resource is available for the processing by acquiring a semaphore
     """
     EDVerbose.DEBUG("Acquire semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
     EDUtilsParallel.semaphoreNbThreadsAcquire()
     EDPlugin.preProcess(self, _edObject)
Ejemplo n.º 10
0
 def finallyProcess(self, _edObject=None):
     """
     after processing of the plugin:
     Release a CPU resource by releasing the semaphore
     """
     self.DEBUG("Release semaphore nbCPU by plugin %s" % self.getPluginName())
     EDUtilsParallel.semaphoreNbThreadsRelease()
     EDPlugin.finallyProcess(self, _edObject)
Ejemplo n.º 11
0
    def __init__(self, _strPluginName, _strPluginDir=None, _strTestName=None):
        """
        Initialize the Plugin Unit test
        """
        EDTestCasePlugin.__init__(self, _strPluginName, _strPluginDir, _strTestName)
        self.__edPlugin = None
#        To track dead locks
        EDUtilsParallel.initializeNbThread(1)
Ejemplo n.º 12
0
 def __init__(self, _strPluginName, _strPluginDir=None, _strTestName=None):
     """
     Initialize the Plugin Unit test
     """
     EDTestCasePlugin.__init__(self, _strPluginName, _strPluginDir,
                               _strTestName)
     self.__edPlugin = None
     #        To track dead locks
     EDUtilsParallel.initializeNbThread(1)
Ejemplo n.º 13
0
 def preProcess(self, _edObject=None):
     """
     preProcess of the plugin:
     Ensure a CPU resource is available for the processing by acquiring a semaphore
     """
     EDVerbose.DEBUG(
         "Acquire semaphore nbCPU by plugin %s, currently value: %s" %
         (self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
     EDUtilsParallel.semaphoreNbThreadsAcquire()
     EDPlugin.preProcess(self, _edObject)
Ejemplo n.º 14
0
 def finallyProcess(self, _edObject=None):
     """
     after processing of the plugin:
     Release a CPU resource by releasing the semaphore
     """
     EDVerbose.DEBUG(
         "Release semaphore nbCPU by plugin %s, currently value: %s" %
         (self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
     EDUtilsParallel.semaphoreNbThreadsRelease()
     EDPlugin.finallyProcess(self, _edObject)
Ejemplo n.º 15
0
 def unitTestDetectNumberOfCPUs(self):
     """
     test the execution of detectNumberOfCPUs
     """
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs()
     EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU)
     EDAssert.equal(types.IntType, type(iNbCPU), "Number of CPU is an integer")
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1
     EDAssert.equal(1, iNbCPU, "Limit number of CPU")
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100
     EDAssert.equal(100, iNbCPU, "Force number of CPU")
Ejemplo n.º 16
0
 def cleanUp(self, _edPlugin):
     """
     Cleans up some empty directories 
     (mainly working directories that were created automatically, but not filled as Unit tests do not execute the plugin)
     """
     workingDirectory = _edPlugin.getWorkingDirectory()
     if (workingDirectory is not None):
         fileList = glob.glob(os.path.join(workingDirectory, "*"))
         if (len(fileList) == 0):
             EDVerbose.DEBUG("Deleting " + workingDirectory + " ...")
             shutil.rmtree(workingDirectory)
     EDUtilsParallel.semaphoreNbThreadsRelease()
Ejemplo n.º 17
0
 def failurePluginExecution(self, _edObject=None):
     """
     Method called when the execution of the plugin failed 
     """
     EDUtilsParallel.semaphoreNbThreadsRelease()
     self.__bIsFirstExecute = False
     if self.__functXMLerr is None:
         EDVerbose.screen("Plugin %s execution ended with failure" % self.__strPluginName)
     else:
         self.__semaphoreErr.acquire()
         self.__functXMLerr(_edObject.dataInput.marshal())
         self.__semaphoreErr.release()
Ejemplo n.º 18
0
 def successPluginExecution(self, _edObject=None):
     """
     Method called when the execution of the plugin succeeds 
     """
     EDUtilsParallel.semaphoreNbThreadsRelease()
     self.__bIsFirstExecute = False
     if self.__functXMLout is None:
         EDVerbose.screen("Plugin %s execution ended with success" % self.__strPluginName)
     else:
         self.__semaphoreOut.acquire()
         self.__functXMLout(_edObject.dataInput.marshal(), _edObject.getDataOutput().marshal())
         self.__semaphoreOut.release()
Ejemplo n.º 19
0
 def unitTestDetectNumberOfCPUs(self):
     """
     test the execution of detectNumberOfCPUs
     """
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs()
     EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU)
     EDAssert.equal(types.IntType, type(iNbCPU),
                    "Number of CPU is an integer")
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1)  #limited to 1
     EDAssert.equal(1, iNbCPU, "Limit number of CPU")
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True)  #forced to 100
     EDAssert.equal(100, iNbCPU, "Force number of CPU")
Ejemplo n.º 20
0
 def cleanUp(self, _edPlugin):
     """
     Cleans up some empty directories 
     (mainly working directories that were created automatically, but not filled as Unit tests do not execute the plugin)
     """
     workingDirectory = _edPlugin.getWorkingDirectory()
     if(workingDirectory is not None):
         fileList = glob.glob(os.path.join(workingDirectory, "*"))
         if(len(fileList) == 0):
             EDVerbose.DEBUG("Deleting " + workingDirectory + " ...")
             shutil.rmtree(workingDirectory)
     EDUtilsParallel.semaphoreNbThreadsRelease()
Ejemplo n.º 21
0
 def executeControlledPlugins(self):
     """
     Execute all plugins under control: 
     """
     bAllFinished = False
     while not bAllFinished:
         #acquire all semaphores to be sure no plugins are under configuration !
         for sem in self.lstSem:
             with sem:
                 pass
         if self.queue.empty():
             self.synchronizePlugins()
             bAllFinished = self.queue.empty()
         else:
             while not self.queue.empty():
                 try:
                     plugin = self.queue.get()
                 except Exception:
                     self.WARNING("In EDPluginControlAlignStackv1_0, exception in self.queue.get()")
                     break
                 else:
                     #this is a hack to prevent thousands of threads to be launched at once.
                     with EDUtilsParallel.getSemaphoreNbThreads():
                         pass
                     plugin.execute()
             self.synchronizePlugins()
Ejemplo n.º 22
0
 def executeControlledPlugins(self):
     """
     Execute all plugins under control: 
     """
     bAllFinished = False
     while not bAllFinished:
         #acquire all semaphores to be sure no plugins are under configuration !
         for sem in self.lstSem:
             with sem:
                 pass
         if self.queue.empty():
             self.synchronizePlugins()
             bAllFinished = self.queue.empty()
         else:
             while not self.queue.empty():
                 try:
                     plugin = self.queue.get()
                 except Exception:
                     self.WARNING(
                         "In EDPluginControlAlignStackv1_0, exception in self.queue.get()"
                     )
                     break
                 else:
                     #this is a hack to prevent thousands of threads to be launched at once.
                     with EDUtilsParallel.getSemaphoreNbThreads():
                         pass
                     plugin.execute()
             self.synchronizePlugins()
Ejemplo n.º 23
0
 def waitForAllProcessToFinish(self):
     """
     as it names says, this method waits for all plug-ins which are currently running to finish before returning.
     """
     sys.stderr.write("Waiting for launched jobs to finish .")
     while (EDUtilsParallel.getNbRunning() > 0):
         time.sleep(1)
         sys.stderr.write(".")
     sys.stderr.write("Done.\n")
Ejemplo n.º 24
0
 def waitForAllProcessToFinish(self):
     """
     as it names says, this method waits for all plug-ins which are currently running to finish before returning.
     """
     sys.stderr.write("Waiting for launched jobs to finish .")
     while (EDUtilsParallel.getNbRunning() > 0):
         time.sleep(1)
         sys.stderr.write(".")
     sys.stderr.write("Done.\n")
Ejemplo n.º 25
0
    def __init__(self, strPluginName, iNbCpu=None):
        EDLogging.__init__(self)
        self.pluginName = strPluginName
        self.startTime = time.time()
        try:
            self.iNbCpu = int(iNbCpu)
        except:
            self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs()

        self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu)
        self.__semaphoreNbThreads = Semaphore(self.iNbCpu)
        EDUtilsParallel.initializeNbThread(self.iNbCpu)
        self.jobQueue = Queue()
        self.processingSem = Semaphore()
        self.statLock = Semaphore()
        self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
        self.lastFailure = "No job Failed (yet)"
        self.lastSuccess = "No job succeeded (yet)"
Ejemplo n.º 26
0
 def join(self):
     """
     wait for all jobs to finish
     """
     while not (self.jobQueue.empty() and \
             (self.__semaphoreNbThreads._Semaphore__value == self.iNbCpu) and \
             (EDUtilsParallel.getNbRunning() == 0) and \
             (self.processingSem._Semaphore__value == 1) and\
             (len(EDStatus.getRunning()) == 0)):
         time.sleep(1)
Ejemplo n.º 27
0
 def __init__(self, _strPluginName, _strPluginDir=None, _strTestName=None):
     """ 
     Initialise the Plugin Execution test
         - Edna site
         - Configuration files
         - Data input file
         - Plugin launcher
     """
     EDTestCasePlugin.__init__(self, _strPluginName, _strPluginDir, _strTestName)
     self._strRefConfigFile = None
     self._dictStrDataInputFiles = {}
     self._strDefaultInputDataKey = "defaultInputData"
     self._dictStrReferenceDataOutputFiles = {}
     self._strDefaultOutputDataKey = "defaultOutputData"
     self._iNoExpectedErrorMessages = 0
     self._iNoExpectedWarningMessages = 0
     self._bAcceptPluginFailure = False
     # Deprecated!
     self.m_edObtainedOutputDataFile = None
     EDUtilsParallel.uninitializeNbThread()
 def preProcess(self):
     """
     PreProcess of the execution test: download a set of images  from http://www.edna-site.org
     and remove any existing output file 
     """
     EDTestCasePluginExecute.preProcess(self)
     self.loadTestImage(["bioSaxsRaw.edf", "bioSaxsMask.edf",
                         "bioSaxsCorrected.edf", "bioSaxsProcessNormalized.edf",
                         "bioSaxsProcessIntegrated.edf", "bioSaxsProcessIntegrated1_2.dat"])
     strExpectedOutput = self.readAndParseFile (self.getReferenceDataOutputFile())
     EDVerbose.DEBUG("strExpectedOutput:" + strExpectedOutput)
     xsDataResultReference = XSDataResultBioSaxsProcessOneFilev1_0.parseString(strExpectedOutput)
     self.refNormImg = xsDataResultReference.normalizedImage.path.value
     self.refIntCrv = xsDataResultReference.integratedCurve.path.value
     if not os.path.isdir(os.path.dirname(self.refNormImg)):
         os.makedirs(os.path.dirname(self.refNormImg))
     if os.path.isfile(self.refNormImg):
         os.remove(self.refNormImg)
     if os.path.isfile(self.refIntCrv):
         os.remove(self.refIntCrv)
     EDUtilsParallel.initializeNbThread()
Ejemplo n.º 29
0
    def chooseWorker(self):
        """
        selects the worker and sets self.worker
        """
#        self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s" % self.getId())
        if EDPluginSPDCorrectv10.__iNumberOfWorker == 0:
            EDPluginSPDCorrectv10.__iNumberOfWorker = EDUtilsParallel.detectNumberOfCPUs(self._iConfigNumberOfWorker)
        self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s nb of workers = %s lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__iNumberOfWorker, EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
#        self.DEBUG("chooseWorker: \t EDPluginSPDCorrectv10.__lock.acquire(), currently: %i" % EDPluginSPDCorrectv10.__lock._Semaphore__value)
        EDPluginSPDCorrectv10.__lockWorker.acquire()
        for oneWorker in EDPluginSPDCorrectv10.__listOfWorker :
            self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Status of worker %i: %s" % (self.getId(), oneWorker.pid, oneWorker.getStatus()))
            if (oneWorker.getConfig() == self._SPDconfig)  and (oneWorker.getStatus() in ["free", "uninitialized"]):
                self.worker = oneWorker
        if self.worker is None:
            if len(EDPluginSPDCorrectv10.__listOfWorker) < EDPluginSPDCorrectv10.__iNumberOfWorker :
                self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Initializing %i th worker" % (self.getId(), len(EDPluginSPDCorrectv10.__listOfWorker)))
                self.worker = SPDworker()
                EDPluginSPDCorrectv10.__listOfWorker.append(self.worker)
                self.worker.setExecutable(self.getExecutable())
                self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
                self.worker.setLogFilename(os.path.join(self.getSPDCommonDirectory(), "worker-%02i.log" % self.workerID))
                self.worker.initialize(self._SPDconfig)
                self.worker.setTimeOut(self.getTimeOut())
        while self.worker is None:
            self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: No workers still to initialize" % self.getId())
            bConfigOK = False
            for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker) :
#                self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), oneWorker.getConfig()))
#                self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), self._SPDconfig))
                #bug1 those string must specifically be the same ... change line 248 in spdCake1.5
                if (oneWorker.getConfig() == self._SPDconfig):
                    bConfigOK = True
                    if (oneWorker.getStatus() in ["free", "uninitialized"]):
                        self.worker = oneWorker
                        self.workerID = idx
            if bConfigOK == False:
                for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker[:]) :
#                    self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: worker %i status %s " % (self.getId(), idx, oneWorker.status))
                    if (oneWorker.getStatus() in ["free", "uninitialized"]):
                        #Bug2 why doest this work ???
#                        self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Resetting worker %i status %s " % (self.getId(), idx, oneWorker.status))
                        oneWorker.initialize(self._SPDconfig)
                        self.worker = oneWorker
                        self.workerID = idx
                        EDPluginSPDCorrectv10.__listOfWorker[idx] = (self.worker)
            self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Left loop worker %s status %s " % (self.getId(), self.workerID, oneWorker))
            time.sleep(0.1)

        if self.workerID is None:
            self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
        self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Release lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
        EDPluginSPDCorrectv10.__lockWorker.release()
Ejemplo n.º 30
0
 def integrate(self):
     if (self.integrator.getPyFAI() != self.integrator_config) or \
        (self.integrator.wavelength != self.experimentSetup.wavelength.value):
         self.screen("Resetting PyFAI integrator")
         self.integrator.setPyFAI(**self.integrator_config)
         self.integrator.wavelength = self.experimentSetup.wavelength.value
     with EDUtilsParallel.getSemaphoreNbThreads():
         img = fabio.open(self.normalizedImage)
         variance = img.next()
         q, I, std = self.integrator.saxs(data=img.data, nbPt=max(img.dim1, img.dim2),
                                    correctSolidAngle=True,
                                    variance=variance.data,
                                    dummy= -2, delta_dummy=1.1,
                                    method="splitBBox")
     return q, I, std
Ejemplo n.º 31
0
 def __init__(self, cl, name):
     EDLogging.__init__(self)
     PyTango.Device_4Impl.__init__(self, cl, name)
     self.init_device()
     if isinstance(iNbCpu, int):
         self.screen("Initializing tangoDS with max %i jobs in parallel." % iNbCpu)
         self.__semaphoreNbThreads = threading.Semaphore(iNbCpu)
     else:
         self.__semaphoreNbThreads = threading.Semaphore(EDUtilsParallel.detectNumberOfCPUs())
     self.jobQueue = Queue()
     self.processingSem = threading.Semaphore()
     self.statLock = threading.Lock()
     self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
     self.lastFailure = "No job Failed (yet)"
     self.lastSuccess = "No job succeeded (yet)"
Ejemplo n.º 32
0
 def integrate(self):
     if (self.integrator.getPyFAI() != self.integrator_config) or \
        (self.integrator.wavelength != self.experimentSetup.wavelength.value):
         self.screen("Resetting PyFAI integrator")
         self.integrator.setPyFAI(**self.integrator_config)
         self.integrator.wavelength = self.experimentSetup.wavelength.value
     with EDUtilsParallel.getSemaphoreNbThreads():
         img = fabio.open(self.normalizedImage)
         variance = img.next()
         q, I, std = self.integrator.saxs(data=img.data, nbPt=max(img.dim1, img.dim2),
                                    correctSolidAngle=True,
                                    variance=variance.data,
                                    dummy= -2, delta_dummy=1.1,
                                    method="splitBBox")
     return q, I, std
Ejemplo n.º 33
0
 def __init__(self, cl, name):
     EDLogging.__init__(self)
     PyTango.Device_4Impl.__init__(self, cl, name)
     self.init_device()
     if isinstance(iNbCpu, int):
         self.screen("Initializing tangoDS with max %i jobs in parallel." %
                     iNbCpu)
         self.__semaphoreNbThreads = threading.Semaphore(iNbCpu)
     else:
         self.__semaphoreNbThreads = threading.Semaphore(
             EDUtilsParallel.detectNumberOfCPUs())
     self.jobQueue = Queue()
     self.processingSem = threading.Semaphore()
     self.statLock = threading.Lock()
     self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
     self.lastFailure = "No job Failed (yet)"
     self.lastSuccess = "No job succeeded (yet)"
Ejemplo n.º 34
0
 def process(self, _edPlugin=None):
     """
     Executes the action cluster. This method will return only when
     all actions have finished.
     """
     self.DEBUG("EDActionCluster.process")
     if (not self.__iNoThreads):
         self.__iNoThreads = EDUtilsParallel.detectNumberOfCPUs()
     self.__semaphoreActionCluster = threading.Semaphore(self.__iNoThreads)
     for edAction in self.__lActions:
         edAction.connectSUCCESS(self.__semaphoreRelease)
         edAction.connectFAILURE(self.__setActionClusterFailure)
         edAction.connectFAILURE(self.__semaphoreRelease)
         self.DEBUG("EDActionCluster.process : Starting action %s" % edAction.getClassName())
         self.__semaphoreActionCluster.acquire()
         edAction.execute()
     # Wait for all threads to finish
     for edAction in self.__lActions:
         edAction.join()
Ejemplo n.º 35
0
 def process(self, _edPlugin=None):
     """
     Executes the action cluster. This method will return only when
     all actions have finished.
     """
     self.DEBUG("EDActionCluster.process")
     if (not self.__iNoThreads):
         self.__iNoThreads = EDUtilsParallel.detectNumberOfCPUs()
     self.__semaphoreActionCluster = threading.Semaphore(self.__iNoThreads)
     for edAction in self.__lActions:
         edAction.connectSUCCESS(self.__semaphoreRelease)
         edAction.connectFAILURE(self.__setActionClusterFailure)
         edAction.connectFAILURE(self.__semaphoreRelease)
         self.DEBUG("EDActionCluster.process : Starting action %s" %
                    edAction.getClassName())
         self.__semaphoreActionCluster.acquire()
         edAction.execute()
     # Wait for all threads to finish
     for edAction in self.__lActions:
         edAction.join()
Ejemplo n.º 36
0
 def __init__(self, cl, name):
     EDLogging.__init__(self)
     PyTango.Device_4Impl.__init__(self, cl, name)
     self.init_device()
     if isinstance(iNbCpu, int):
         self.screen("Initializing tangoDS with max %i jobs in parallel." % iNbCpu)
         self.__semaphoreNbThreads = threading.Semaphore(iNbCpu)
     else:
         self.__semaphoreNbThreads = threading.Semaphore(EDUtilsParallel.detectNumberOfCPUs())
     self.quit = False
     self.jobQueue = Queue()  # queue containing jobs to process
     self.eventQueue = Queue()  # queue containing finished jobs
     self.statLock = threading.Lock()
     self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
     self.lastFailure = "No job Failed (yet)"
     self.lastSuccess = "No job succeeded (yet)"
     self.processingThread = threading.Thread(target=self.startProcessing)
     self.processingThread.start()
     self.finishingThread = threading.Thread(target=self.process_event)
     self.finishingThread.start()
Ejemplo n.º 37
0
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlDozorv1_0.process")
        self.sendMessageToMXCuBE("Processing started...", "info")
        EDUtilsParallel.initializeNbThread()
        xsDataResultControlDozor = XSDataResultControlDozor()
        # Check if connection to ISPyB needed
        if self.dataInput.dataCollectionId is not None:
            edPluginRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = edPluginRetrieveDataCollection.dataOutput.dataCollection
            if self.batchSize is None:
                batchSize = ispybDataCollection.numberOfImages
            else:
                batchSize = self.batchSize
            if batchSize > self.maxBatchSize:
                batchSize = self.maxBatchSize
            if abs(ispybDataCollection.overlap) > 1:
                self.hasOverlap = True
                self.overlap = ispybDataCollection.overlap
            dictImage = self.createImageDictFromISPyB(ispybDataCollection)
        else:
            # No connection to ISPyB, take parameters from input
            if self.dataInput.batchSize is None:
                batchSize = self.maxBatchSize
            else:
                batchSize = self.dataInput.batchSize.value
            dictImage = self.createImageDict(self.dataInput)
        self.screen("Dozor batch size: {0}".format(batchSize))
        if self.dataInput.hdf5BatchSize is not None:
            self.hdf5BatchSize = self.dataInput.hdf5BatchSize.value
        listAllBatches = self.createListOfBatches(dictImage.keys(), batchSize)
        if dictImage[listAllBatches[0][0]].path.value.endswith("h5"):
            # Convert HDF5 images to CBF
            self.screen("HDF5 converter batch size: {0}".format(self.batchSize))
            if self.doRadiationDamage:
                self.cbfTempDir = None
            else:
                self.cbfTempDir = tempfile.mkdtemp(prefix="CbfTemp_")
            listHdf5Batches = self.createListOfBatches(dictImage.keys(), self.batchSize)
            dictImage, self.hasHdf5Prefix = self.convertToCBF(dictImage, listHdf5Batches, self.doRadiationDamage)
        for listBatch in listAllBatches:
            # Read the header from the first image in the batch
            xsDataFile = dictImage[listBatch[0]]
            edPluginControlReadImageHeader = self.loadPlugin(self.strEDPluginControlReadImageHeaderName)
            xsDataInputReadImageHeader = XSDataInputReadImageHeader()
            xsDataInputReadImageHeader.image = xsDataFile
            edPluginControlReadImageHeader.dataInput = xsDataInputReadImageHeader
            edPluginControlReadImageHeader.executeSynchronous()
            subWedge = edPluginControlReadImageHeader.dataOutput.subWedge
            xsDataInputDozor = XSDataInputDozor()
            beam = subWedge.experimentalCondition.beam
            detector = subWedge.experimentalCondition.detector
            goniostat = subWedge.experimentalCondition.goniostat
            xsDataInputDozor.detectorType = detector.type
            xsDataInputDozor.exposureTime = XSDataDouble(beam.exposureTime.value)
            xsDataInputDozor.spotSize = XSDataInteger(3)
            xsDataInputDozor.detectorDistance = XSDataDouble(detector.distance.value)
            xsDataInputDozor.wavelength = XSDataDouble(beam.wavelength.value)
#            xsDataInputDozor.fractionPolatization : XSDataDouble optional
            orgx = detector.beamPositionY.value / detector.pixelSizeY.value
            orgy = detector.beamPositionX.value / detector.pixelSizeX.value
            xsDataInputDozor.orgx = XSDataDouble(orgx)
            xsDataInputDozor.orgy = XSDataDouble(orgy)
            xsDataInputDozor.oscillationRange = XSDataDouble(goniostat.oscillationWidth.value)
#            xsDataInputDozor.imageStep : XSDataDouble optional
            xsDataInputDozor.startingAngle = XSDataDouble(goniostat.rotationAxisStart.value)
            xsDataInputDozor.firstImageNumber = subWedge.image[0].number
            xsDataInputDozor.numberImages = XSDataInteger(len(listBatch))
            if self.hasOverlap:
                xsDataInputDozor.overlap = XSDataAngle(self.overlap)
            strFileName = subWedge.image[0].path.value
            strPrefix = EDUtilsImage.getPrefix(strFileName)
            strSuffix = EDUtilsImage.getSuffix(strFileName)
            if EDUtilsPath.isEMBL():
                strXDSTemplate = "%s_?????.%s" % (strPrefix, strSuffix)
            elif self.hasHdf5Prefix and not self.hasOverlap:
                strXDSTemplate = "%s_??????.%s" % (strPrefix, strSuffix)
            else:
                strXDSTemplate = "%s_????.%s" % (strPrefix, strSuffix)
            xsDataInputDozor.nameTemplateImage = XSDataString(os.path.join(os.path.dirname(strFileName), strXDSTemplate))
            xsDataInputDozor.wedgeNumber = self.dataInput.wedgeNumber
            xsDataInputDozor.radiationDamage = self.dataInput.radiationDamage
            edPluginDozor = self.loadPlugin(self.strEDPluginDozorName, "Dozor_%05d" % subWedge.image[0].number.value)
            edPluginDozor.dataInput = xsDataInputDozor
            edPluginDozor.execute()
            edPluginDozor.synchronize()
            indexImage = 0
            imageDozorBatchList = []

            for xsDataResultDozor in edPluginDozor.dataOutput.imageDozor:
                xsDataControlImageDozor = XSDataControlImageDozor()
                xsDataControlImageDozor.number = xsDataResultDozor.number
                xsDataControlImageDozor.image = dictImage[listBatch[indexImage]]
                xsDataControlImageDozor.spotsNumOf = xsDataResultDozor.spotsNumOf
                xsDataControlImageDozor.spotsIntAver = xsDataResultDozor.spotsIntAver
                xsDataControlImageDozor.spotsResolution = xsDataResultDozor.spotsResolution
                xsDataControlImageDozor.powderWilsonScale = xsDataResultDozor.powderWilsonScale
                xsDataControlImageDozor.powderWilsonBfactor = xsDataResultDozor.powderWilsonBfactor
                xsDataControlImageDozor.powderWilsonResolution = xsDataResultDozor.powderWilsonResolution
                xsDataControlImageDozor.powderWilsonCorrelation = xsDataResultDozor.powderWilsonCorrelation
                xsDataControlImageDozor.powderWilsonRfactor = xsDataResultDozor.powderWilsonRfactor
                xsDataControlImageDozor.mainScore = xsDataResultDozor.mainScore
                xsDataControlImageDozor.spotScore = xsDataResultDozor.spotScore
                xsDataControlImageDozor.visibleResolution = xsDataResultDozor.visibleResolution
                xsDataControlImageDozor.spotFile = xsDataResultDozor.spotFile
                xsDataControlImageDozor.angle = xsDataResultDozor.angle
                xsDataResultControlDozor.addImageDozor(xsDataControlImageDozor)
                if xsDataResultControlDozor.inputDozor is None:
                    xsDataResultControlDozor.inputDozor = XSDataDozorInput().parseString(xsDataInputDozor.marshal())
                indexImage += 1

                dozorSpotListShape = []
                dozorSpotList = []
                spotFile = None
                if xsDataControlImageDozor.spotFile is not None:
                    spotFile = xsDataControlImageDozor.spotFile.path.value
                    if os.path.exists(spotFile):
                        numpyArray = numpy.loadtxt(spotFile, skiprows=3)
                        dozorSpotList = base64.b64encode(numpyArray.tostring())
                        dozorSpotListShape.append(numpyArray.shape[0])
                        if len(numpyArray.shape) > 1:
                            dozorSpotListShape.append(numpyArray.shape[1])

                imageDozorDict = {"index": xsDataControlImageDozor.number.value,
                                  "imageName": xsDataControlImageDozor.image.path.value,
                                  "dozor_score": xsDataControlImageDozor.mainScore.value,
                                  "dozorSpotsNumOf" : xsDataControlImageDozor.spotsNumOf.value,
                                  "dozorSpotFile": spotFile,
                                  "dozorSpotList" : dozorSpotList,
                                  "dozorSpotListShape": dozorSpotListShape,
                                  "dozorSpotsIntAver": xsDataControlImageDozor.spotsIntAver.value,
                                  "dozorSpotsResolution": xsDataControlImageDozor.spotsResolution.value
                                  }
                imageDozorBatchList.append(imageDozorDict)

            xsDataResultControlDozor.halfDoseTime = edPluginDozor.dataOutput.halfDoseTime
            xsDataResultControlDozor.pngPlots = edPluginDozor.dataOutput.pngPlots

            self.sendResultToMXCuBE(imageDozorBatchList)
            self.sendMessageToMXCuBE("Batch processed")
        self.dataOutput = xsDataResultControlDozor
        if self.cbfTempDir is not None:
            if self.dataInput.keepCbfTmpDirectory is not None and self.dataInput.keepCbfTmpDirectory.value:
                self.dataOutput.pathToCbfDirectory = XSDataFile(XSDataString(self.cbfTempDir))
            else:
                shutil.rmtree(self.cbfTempDir)
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.process")
        EDUtilsParallel.initializeNbThread()
        # Check batch size
        if self.dataInput.batchSize is None:
            batchSize = 1
        else:
            batchSize = self.dataInput.batchSize.value
        self.screen("Batch size: {0}".format(batchSize))
        # Check if we should do distlSignalStrength:
        bDoDistlSignalStrength = True
        if self.dataInput.doDistlSignalStrength is not None:
            if not self.dataInput.doDistlSignalStrength.value:
                bDoDistlSignalStrength = False
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                 bDoIndexing = True
        # Loop through all the incoming reference images
        listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators()
        listPluginDistl = []
        listPluginDozor = []
        listBatch = []
        indexBatch = 0
        listH5FilePath = []
        ispybDataCollection = None
        for xsDataImage in listXSDataImage:
            strPathToImage = xsDataImage.path.value
            # If Eiger, just wait for the h5 file
            if "id30a3" in strPathToImage:
                h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage, batchSize)
#                print(h5FilePath)
#                print(hdf5ImageNumber)
                if not h5DataFilePath in listH5FilePath:
                    self.screen("ID30a3 Eiger data, waiting for master and data files...")
                    listH5FilePath.append(h5DataFilePath)
                    self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                    xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath))
                    xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
                    xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                    self.screen("Waiting for file {0}".format(h5DataFilePath))
                    self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
                    self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                    self.edPluginMXWaitFile.executeSynchronous()
#                    hdf5FilePath = strPathToImage.replace(".cbf", ".h5")
                    ispybDataCollection = None
                    time.sleep(1)
                indexLoop = 1
                continueLoop = True
                while continueLoop:
                    xsDataInputControlH5ToCBF = XSDataInputControlH5ToCBF()
                    xsDataInputControlH5ToCBF.hdf5File = XSDataFile(XSDataString(strPathToImage))
                    imageNumber = EDUtilsImage.getImageNumber(strPathToImage)
                    xsDataInputControlH5ToCBF.imageNumber = XSDataInteger(imageNumber)
                    xsDataInputControlH5ToCBF.hdf5ImageNumber = XSDataInteger(hdf5ImageNumber)
                    xsDataInputControlH5ToCBF.ispybDataCollection = ispybDataCollection
                    edPluginControlH5ToCBF = self.loadPlugin(self.strPluginControlH5ToCBF, "ControlH5ToCBF_%04d_%02d" % (imageNumber, indexLoop))
                    edPluginControlH5ToCBF.dataInput = xsDataInputControlH5ToCBF
                    edPluginControlH5ToCBF.executeSynchronous()
                    cbfFile = edPluginControlH5ToCBF.dataOutput.outputCBFFile
#                    print(cbfFile)
#                    print(indexLoop)
                    if cbfFile is not None:
                        strPathToImage = cbfFile.path.value
#                        print(cbfFile.path.value)
                        if os.path.exists(strPathToImage):
                            self.screen("Image has been converted to CBF file: {0}".format(strPathToImage))
                            continueLoop = False
#                    print(continueLoop)
                    if continueLoop:
                        self.screen("Still waiting for converting to CBF file: {0}".format(strPathToImage))
                        indexLoop += 1
                        time.sleep(5)
                        if indexLoop > 10:
                            continueLoop = False

                ispybDataCollection = edPluginControlH5ToCBF.dataOutput.ispybDataCollection


            elif not os.path.exists(strPathToImage):
                self.screen("Waiting for file {0}".format(strPathToImage))
                self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage))
                xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
                xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
                self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(strPathToImage):
                strError = "Time-out while waiting for image %s" % strPathToImage
                self.error(strError)
                self.addErrorMessage(strError)
                self.setFailure()
            else:
                # Check if we should run distl.signalStrength
                xsDataImageNew = XSDataImage(XSDataString(strPathToImage))
                edPluginPluginExecImageQualityIndicator = None
                if bDoDistlSignalStrength:
                    if self.bUseThinClient:
                        strPluginName = self.strPluginNameThinClient
                    else:
                        strPluginName = self.strPluginName
                    edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName)
                    self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator)
                    xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength()
                    xsDataInputDistlSignalStrength.setReferenceImage(xsDataImageNew)
                    edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength)
                    edPluginPluginExecImageQualityIndicator.execute()
                listPluginDistl.append((xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator))
                listBatch.append(xsDataImageNew.copy())
                if len(listBatch) == batchSize:
                    edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
                    xsDataInputControlDozor = XSDataInputControlDozor()
                    for image in listBatch:
                        xsDataInputControlDozor.addImage(XSDataFile(image.path))
                    xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
                    edPluginControlDozor.dataInput = xsDataInputControlDozor
                    edPluginControlDozor.execute()
                    listPluginDozor.append((edPluginControlDozor, listBatch))
                    listBatch = []
        if len(listBatch) > 0:
            # Process the remaining images...
            edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
            xsDataInputControlDozor = XSDataInputControlDozor()
            for image in listBatch:
                xsDataInputControlDozor.addImage(XSDataFile(image.path))
            xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
            edPluginControlDozor.dataInput = xsDataInputControlDozor
            edPluginControlDozor.execute()
            listPluginDozor.append([edPluginControlDozor, listBatch])
        listIndexing = []
        # Synchronize all image quality indicator plugins and upload to ISPyB
        xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators()

        for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl:
            xsDataImageQualityIndicators = XSDataImageQualityIndicators()
            xsDataImageQualityIndicators.image = xsDataImage.copy()
            if edPluginPluginExecImageQualityIndicator is not None:
                edPluginPluginExecImageQualityIndicator.synchronize()
                if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
                    xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
                            edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
            self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)

        for (edPluginControlDozor, listBatch) in listPluginDozor:
            edPluginControlDozor.synchronize()
            for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
                for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                    if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
                        xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
                        xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
                        if imageDozor.spotFile is not None:
                            if os.path.exists(imageDozor.spotFile.path.value):
                                numpyArray = numpy.loadtxt(imageDozor.spotFile.path.value, skiprows=3)
                                xsDataImageQualityIndicators.dozorSpotList = XSDataString(base64.b64encode(numpyArray.tostring()))
                                xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[0]))
                                if len(numpyArray.shape) > 1:
                                    xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[1]))
                        xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
                        xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
                        if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
                            if edPluginControlDozor.dataOutput.inputDozor is not None:
                                self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString(
                                               edPluginControlDozor.dataOutput.inputDozor.marshal())
            if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
                xsDataISPyBImageQualityIndicators = \
                    XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
                xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators)
#        print xsDataInputStoreListOfImageQualityIndicators.marshal()
        if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
            self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
            self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
            self.edPluginISPyB.execute()
        #
        if bDoIndexing:
            # Find the 5 most intensive images (TIS):
            listImage = []
            # Check that we have dozor_score from all images:
            has_dozor_score = True
            for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                if imageQualityIndicators.dozor_score is None:
                    has_dozor_score = False
            if has_dozor_score:
                listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
                                    key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value)
            else:
                listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
                                    key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value)
            for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
                if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
                    xsDataInputReadImageHeader = XSDataInputReadImageHeader()
                    xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path)
                    self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName)
                    self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
                    self.edPluginReadImageHeader.executeSynchronous()
                    xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
                    if xsDataResultReadImageHeader is not None:
                        xsDataSubWedge = xsDataResultReadImageHeader.subWedge
                        self.xsDataCollection = XSDataCollection()
                        self.xsDataCollection.addSubWedge(xsDataSubWedge)
                        xsDataIndexingInput = XSDataIndexingInput()
                        xsDataIndexingInput.setDataCollection(self.xsDataCollection)
                        xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(xsDataIndexingInput)
                        edPluginMOSFLMIndexing = self.loadPlugin(self.strIndexingMOSFLMPluginName)
                        self.listPluginMOSFLM.append([edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator])
                        edPluginMOSFLMIndexing.setDataInput(xsDataMOSFLMIndexingInput)
                        edPluginMOSFLMIndexing.execute()
            for tupleMOSFLM in self.listPluginMOSFLM:
                edPluginMOSFLMIndexing = tupleMOSFLM[0]
                xsDataResultControlImageQualityIndicator = tupleMOSFLM[1]
                edPluginMOSFLMIndexing.synchronize()
                if not edPluginMOSFLMIndexing.isFailure():
                    xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput
                    xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(xsDataMOSFLMOutput)
                    selectedSolution = xsDataIndexingResult.selectedSolution
                    if selectedSolution is not None:
                        xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
        EDUtilsParallel.initializeNbThread()
        # Check batch size
        if self.dataInput.batchSize is None:
            batchSize = 1
        else:
            batchSize = self.dataInput.batchSize.value
        # Check if we should do distlSignalStrength:
        bDoDistlSignalStrength = True
        if self.dataInput.doDistlSignalStrength is not None:
            if not self.dataInput.doDistlSignalStrength.value:
                bDoDistlSignalStrength = False
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                 bDoIndexing = True
        # Loop through all the incoming reference images
        listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators()
        listPluginDistl = []
        listPluginDozor = []
        listBatch = []
        indexBatch = 0
        for xsDataImage in listXSDataImage:
            self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
            xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
            xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
            xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
            self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
            self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
            self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(xsDataImage.path.value):
                self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
                xsDataInputMXWaitFile.setSize(XSDataInteger(5000000))
                xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
                self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(xsDataImage.path.value):
                strError = "Time-out while waiting for image %s" % xsDataImage.path.value
                self.error(strError)
                self.addErrorMessage(strError)
                self.setFailure()
            else:
                # Check if we should run distl.signalStrength
                edPluginPluginExecImageQualityIndicator = None
                if bDoDistlSignalStrength:
                    if self.bUseThinClient:
                        strPluginName = self.strPluginNameThinClient
                    else:
                        strPluginName = self.strPluginName
                    edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName)
                    self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator)
                    xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength()
                    xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage)
                    edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength)
                    edPluginPluginExecImageQualityIndicator.execute()
                listPluginDistl.append((xsDataImage.copy(), edPluginPluginExecImageQualityIndicator))
                listBatch.append(xsDataImage.copy())
                if len(listBatch) == batchSize:
                    edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
                    xsDataInputControlDozor = XSDataInputControlDozor()
                    for image in listBatch:
                        xsDataInputControlDozor.addImage(XSDataFile(image.path))
                    xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
                    edPluginControlDozor.dataInput = xsDataInputControlDozor
                    edPluginControlDozor.execute()
                    listPluginDozor.append((edPluginControlDozor, listBatch))
                    listBatch = []
        if len(listBatch) > 0:
            # Process the remaining images...
            edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
            xsDataInputControlDozor = XSDataInputControlDozor()
            for image in listBatch:
                xsDataInputControlDozor.addImage(XSDataFile(image.path))
            xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
            edPluginControlDozor.dataInput = xsDataInputControlDozor
            edPluginControlDozor.execute()
            listPluginDozor.append([edPluginControlDozor, listBatch])
        listIndexing = []
        # Synchronize all image quality indicator plugins and upload to ISPyB
        xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators()

        for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl:
            xsDataImageQualityIndicators = XSDataImageQualityIndicators()
            xsDataImageQualityIndicators.image = xsDataImage.copy()
            if edPluginPluginExecImageQualityIndicator is not None:
                edPluginPluginExecImageQualityIndicator.synchronize()
                if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
                    xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
                            edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
            self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)

        for (edPluginControlDozor, listBatch) in listPluginDozor:
            edPluginControlDozor.synchronize()
            for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
                for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                    if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
                        xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
                        xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
                        xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
                        xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
                        if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
                            if edPluginControlDozor.dataOutput.inputDozor is not None:
                                self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString(
                                               edPluginControlDozor.dataOutput.inputDozor.marshal())
            if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
                xsDataISPyBImageQualityIndicators = \
                    XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
                xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators)
#        print xsDataInputStoreListOfImageQualityIndicators.marshal()
        if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
            self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
            self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
            self.edPluginISPyB.execute()
        #
        if bDoIndexing:
            # Find the 5 most intensive images (TIS):
            listImage = []
            # Check that we have dozor_score from all images:
            has_dozor_score = True
            for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                if imageQualityIndicators.dozor_score is None:
                    has_dozor_score = False
            if has_dozor_score:
                listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
                                    key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value)
            else:
                listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
                                    key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value)
            for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
                if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
                    xsDataInputReadImageHeader = XSDataInputReadImageHeader()
                    xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path)
                    self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName)
                    self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
                    self.edPluginReadImageHeader.executeSynchronous()
                    xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
                    if xsDataResultReadImageHeader is not None:
                        edPluginLabelitIndexing = self.loadPlugin(self.strIndexingLabelitPluginName)
                        xsDataInputLabelitIndexing = XSDataInputLabelitIndexing()
                        xsDataInputLabelitIndexing.image.append(XSDataImage(xsDataResultControlImageQualityIndicator.image.path))
                        edPluginLabelitIndexing.setDataInput(xsDataInputLabelitIndexing)
                        self.listPluginLabelit.append([edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator])
                        edPluginLabelitIndexing.execute()
            for tupleLabelit in self.listPluginLabelit:
                edPluginLabelitIndexing = tupleLabelit[0]
                xsDataResultControlImageQualityIndicator = tupleLabelit[1]
                edPluginLabelitIndexing.synchronize()
                if not edPluginLabelitIndexing.isFailure() and edPluginLabelitIndexing.dataOutput is not None:
                    xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput()
                    xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(xsDataResultLabelitIndexing)
                    selectedSolution = xsDataIndexingResult.selectedSolution
                    if selectedSolution is not None:
                        xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
        EDUtilsParallel.initializeNbThread()
        # Check batch size
        if self.dataInput.batchSize is None:
            batchSize = 1
        else:
            batchSize = self.dataInput.batchSize.value
        self.screen(
            "Image quality indicators batch size: {0}".format(batchSize))
        # Check if we should do distlSignalStrength:
        bDoDistlSignalStrength = True
        if self.dataInput.doDistlSignalStrength is not None:
            if not self.dataInput.doDistlSignalStrength.value:
                bDoDistlSignalStrength = False
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                bDoIndexing = True
        # Check if fast mesh (for HDF5)
        isFastMesh = False
        if self.dataInput.fastMesh:
            isFastMesh = self.dataInput.fastMesh.value
        # Loop through all the incoming reference images
        if len(self.dataInput.image) == 0:
            directory = self.dataInput.directory.path.value
            template = self.dataInput.template.value
            startNo = self.dataInput.startNo.value
            endNo = self.dataInput.endNo.value
            listXSDataImage = []
            for index in range(startNo, endNo + 1):
                imageName = template.replace("####", "{0:04d}".format(index))
                imagePath = os.path.join(directory, imageName)
                xsDataImage = XSDataImage(path=XSDataString(imagePath),
                                          number=XSDataInteger(index))
                listXSDataImage.append(xsDataImage)
        else:
            listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators(
        )
        listPluginDistl = []
        listPluginDozor = []
        listOfImagesInBatch = []
        listOfAllBatches = []
        indexBatch = 0
        listH5FilePath = []
        # Process data in batches
        for xsDataImage in listXSDataImage:
            listOfImagesInBatch.append(xsDataImage.copy())
            if len(listOfImagesInBatch) == batchSize:
                listOfAllBatches.append(listOfImagesInBatch)
                listOfImagesInBatch = []
        if len(listOfImagesInBatch) > 0:
            listOfAllBatches.append(listOfImagesInBatch)
            listOfImagesInBatch = []
        # Loop over batches
        for listOfImagesInBatch in listOfAllBatches:
            # First wait for images
            for image in listOfImagesInBatch:
                strPathToImage = image.path.value
                # If Eiger, just wait for the h5 file
                if strPathToImage.endswith(".h5"):
                    h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(
                        strPathToImage,
                        batchSize=batchSize,
                        isFastMesh=isFastMesh)
                    #                print(h5FilePath)
                    #                print(hdf5ImageNumber)
                    if not h5DataFilePath in listH5FilePath:
                        self.screen(
                            "ID30a3 Eiger data, waiting for master and data files..."
                        )
                        listH5FilePath.append(h5DataFilePath)
                        self.edPluginMXWaitFile = self.loadPlugin(
                            self.strPluginMXWaitFileName)
                        xsDataInputMXWaitFile.file = XSDataFile(
                            XSDataString(h5DataFilePath))
                        xsDataInputMXWaitFile.setSize(
                            XSDataInteger(self.minImageSize))
                        xsDataInputMXWaitFile.setTimeOut(
                            XSDataTime(self.fMXWaitFileTimeOut))
                        self.screen(
                            "Waiting for file {0}".format(h5DataFilePath))
                        self.DEBUG("Wait file timeOut set to %f" %
                                   self.fMXWaitFileTimeOut)
                        self.edPluginMXWaitFile.setDataInput(
                            xsDataInputMXWaitFile)
                        self.edPluginMXWaitFile.executeSynchronous()
                        #                    hdf5FilePath = strPathToImage.replace(".cbf", ".h5")
                        time.sleep(1)
                    if not os.path.exists(h5DataFilePath):
                        strError = "Time-out while waiting for image %s" % h5DataFilePath
                        self.error(strError)
                        self.addErrorMessage(strError)
                        self.setFailure()
                else:
                    if not os.path.exists(strPathToImage):
                        # self.screen("Waiting for file {0}".format(strPathToImage))
                        self.edPluginMXWaitFile = self.loadPlugin(
                            self.strPluginMXWaitFileName)
                        xsDataInputMXWaitFile.file = XSDataFile(
                            XSDataString(strPathToImage))
                        xsDataInputMXWaitFile.setSize(
                            XSDataInteger(self.minImageSize))
                        xsDataInputMXWaitFile.setTimeOut(
                            XSDataTime(self.fMXWaitFileTimeOut))
                        self.screen("Wait file timeOut set to %.0f s" %
                                    self.fMXWaitFileTimeOut)
                        self.edPluginMXWaitFile.setDataInput(
                            xsDataInputMXWaitFile)
                        self.edPluginMXWaitFile.executeSynchronous()
                    if not os.path.exists(strPathToImage):
                        strError = "Time-out while waiting for image %s" % strPathToImage
                        self.error(strError)
                        self.addErrorMessage(strError)
                        self.setFailure()
            if not self.isFailure():
                strPathToFirstImage = listOfImagesInBatch[0].path.value
                if strPathToImage.endswith(".h5"):
                    indexLoop = 1
                    continueLoop = True
                    while continueLoop:
                        directory = os.path.dirname(strPathToFirstImage)
                        firstImage = EDUtilsImage.getImageNumber(
                            listOfImagesInBatch[0].path.value)
                        lastImage = EDUtilsImage.getImageNumber(
                            listOfImagesInBatch[-1].path.value)
                        xsDataInputH5ToCBF = XSDataInputH5ToCBF()
                        xsDataInputH5ToCBF.hdf5File = XSDataFile(
                            listOfImagesInBatch[0].path)
                        xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1)
                        xsDataInputH5ToCBF.startImageNumber = XSDataInteger(
                            firstImage)
                        xsDataInputH5ToCBF.endImageNumber = XSDataInteger(
                            lastImage)
                        xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile(
                            XSDataString(directory))
                        edPluginH5ToCBF = self.loadPlugin(
                            "EDPluginH5ToCBFv1_1")
                        edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF
                        edPluginH5ToCBF.execute()
                        edPluginH5ToCBF.synchronize()
                        outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate
                        if outputCBFFileTemplate is not None:
                            lastCbfFile = outputCBFFileTemplate.path.value.replace(
                                "######", "{0:06d}".format(
                                    EDUtilsImage.getImageNumber(
                                        listOfImagesInBatch[-1].path.value)))
                            strPathToImage = os.path.join(
                                directory, lastCbfFile)
                            #                        print(cbfFile.path.value)
                            if os.path.exists(strPathToImage):
                                # Rename all images
                                for image in listOfImagesInBatch:
                                    image.path.value = image.path.value.replace(
                                        ".h5", ".cbf")
                                    imageNumber = EDUtilsImage.getImageNumber(
                                        image.path.value)
                                    oldPath = os.path.join(
                                        directory,
                                        outputCBFFileTemplate.path.value.
                                        replace("######",
                                                "{0:06d}".format(imageNumber)))
                                    newPath = os.path.join(
                                        directory,
                                        outputCBFFileTemplate.path.value.
                                        replace("######",
                                                "{0:04d}".format(imageNumber)))
                                    os.rename(oldPath, newPath)
                                lastCbfFile = outputCBFFileTemplate.path.value.replace(
                                    "######", "{0:04d}".format(
                                        EDUtilsImage.getImageNumber(
                                            listOfImagesInBatch[-1].path.value)
                                    ))
                                strPathToImage = os.path.join(
                                    directory, lastCbfFile)
                                self.screen(
                                    "Image has been converted to CBF file: {0}"
                                    .format(strPathToImage))
                                continueLoop = False
    #                    print(continueLoop)
                        if continueLoop:
                            self.screen(
                                "Still waiting for converting to CBF file: {0}"
                                .format(strPathToImage))
                            indexLoop += 1
                            time.sleep(5)
                            if indexLoop > 10:
                                continueLoop = False

                for image in listOfImagesInBatch:
                    strPathToImage = image.path.value
                    # Check if we should run distl.signalStrength
                    xsDataImageNew = XSDataImage(XSDataString(strPathToImage))
                    xsDataImageNew.number = XSDataInteger(
                        EDUtilsImage.getImageNumber(image.path.value))
                    edPluginPluginExecImageQualityIndicator = None
                    if bDoDistlSignalStrength:
                        if self.bUseThinClient:
                            strPluginName = self.strPluginNameThinClient
                        else:
                            strPluginName = self.strPluginName
                        edPluginPluginExecImageQualityIndicator = self.loadPlugin(
                            strPluginName)
                        self.listPluginExecImageQualityIndicator.append(
                            edPluginPluginExecImageQualityIndicator)
                        xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength(
                        )
                        xsDataInputDistlSignalStrength.setReferenceImage(
                            xsDataImageNew)
                        edPluginPluginExecImageQualityIndicator.setDataInput(
                            xsDataInputDistlSignalStrength)
                        edPluginPluginExecImageQualityIndicator.execute()
                    listPluginDistl.append(
                        (xsDataImageNew.copy(),
                         edPluginPluginExecImageQualityIndicator))

                edPluginControlDozor = self.loadPlugin(
                    self.strPluginNameControlDozor, "ControlDozor_{0}".format(
                        os.path.splitext(
                            os.path.basename(strPathToFirstImage))[0]))
                xsDataInputControlDozor = XSDataInputControlDozor()
                for image in listOfImagesInBatch:
                    xsDataInputControlDozor.addImage(XSDataFile(image.path))
                xsDataInputControlDozor.batchSize = XSDataInteger(
                    len(listOfImagesInBatch))
                edPluginControlDozor.dataInput = xsDataInputControlDozor
                edPluginControlDozor.execute()
                listPluginDozor.append(
                    (edPluginControlDozor, list(listOfImagesInBatch)))

        if not self.isFailure():
            listIndexing = []
            # Synchronize all image quality indicator plugins and upload to ISPyB
            xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators(
            )

            for (xsDataImage,
                 edPluginPluginExecImageQualityIndicator) in listPluginDistl:
                xsDataImageQualityIndicators = XSDataImageQualityIndicators()
                xsDataImageQualityIndicators.image = xsDataImage.copy()
                if edPluginPluginExecImageQualityIndicator is not None:
                    edPluginPluginExecImageQualityIndicator.synchronize()
                    if edPluginPluginExecImageQualityIndicator.dataOutput is not None:
                        if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
                            xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
                                    edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
                self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(
                    xsDataImageQualityIndicators)

            for (edPluginControlDozor, listBatch) in listPluginDozor:
                edPluginControlDozor.synchronize()
                # Check that we got at least one result
                if len(edPluginControlDozor.dataOutput.imageDozor) == 0:
                    # Run the dozor plugin again, this time synchronously
                    firstImage = os.path.basename(listBatch[0].path.value)
                    lastImage = os.path.basename(listBatch[-1].path.value)
                    self.screen(
                        "No dozor results! Re-executing Dozor for images {0} to {1}"
                        .format(firstImage, lastImage))
                    time.sleep(5)
                    edPluginControlDozor = self.loadPlugin(
                        self.strPluginNameControlDozor,
                        "ControlDozor_reexecution_{0}".format(
                            os.path.splitext(firstImage)[0]))
                    xsDataInputControlDozor = XSDataInputControlDozor()
                    for image in listBatch:
                        xsDataInputControlDozor.addImage(XSDataFile(
                            image.path))
                    xsDataInputControlDozor.batchSize = XSDataInteger(
                        batchSize)
                    edPluginControlDozor.dataInput = xsDataInputControlDozor
                    edPluginControlDozor.executeSynchronous()
                for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
                    for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                        if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
                            xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
                            xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
                            if imageDozor.spotFile is not None:
                                if os.path.exists(
                                        imageDozor.spotFile.path.value):
                                    numpyArray = numpy.loadtxt(
                                        imageDozor.spotFile.path.value,
                                        skiprows=3)
                                    xsDataImageQualityIndicators.dozorSpotList = XSDataString(
                                        base64.b64encode(
                                            numpyArray.tostring()))
                                    xsDataImageQualityIndicators.addDozorSpotListShape(
                                        XSDataInteger(numpyArray.shape[0]))
                                    if len(numpyArray.shape) > 1:
                                        xsDataImageQualityIndicators.addDozorSpotListShape(
                                            XSDataInteger(numpyArray.shape[1]))
                            xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
                            xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
                            xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution
                            if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
                                if edPluginControlDozor.dataOutput.inputDozor is not None:
                                    self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput(
                                    ).parseString(
                                        edPluginControlDozor.dataOutput.
                                        inputDozor.marshal())
                if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
                    xsDataISPyBImageQualityIndicators = \
                        XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
                    xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(
                        xsDataISPyBImageQualityIndicators)

    #        print xsDataInputStoreListOfImageQualityIndicators.marshal()
            if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
                self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
                self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
                self.edPluginISPyB.execute()
            #
            if bDoIndexing:
                # Find the 5 most intensive images (TIS):
                listImage = []
                # Check that we have dozor_score from all images:
                has_dozor_score = True
                for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                    if imageQualityIndicators.dozor_score is None:
                        has_dozor_score = False
                if has_dozor_score:
                    listSorted = sorted(
                        self.xsDataResultControlImageQualityIndicators.
                        imageQualityIndicators,
                        key=lambda imageQualityIndicators:
                        imageQualityIndicators.dozor_score.value)
                else:
                    listSorted = sorted(
                        self.xsDataResultControlImageQualityIndicators.
                        imageQualityIndicators,
                        key=lambda imageQualityIndicators:
                        imageQualityIndicators.totalIntegratedSignal.value)
                for xsDataResultControlImageQualityIndicator in listSorted[
                        -5:]:
                    if xsDataResultControlImageQualityIndicator.dozor_score.value > 1:
                        xsDataInputReadImageHeader = XSDataInputReadImageHeader(
                        )
                        xsDataInputReadImageHeader.image = XSDataFile(
                            xsDataResultControlImageQualityIndicator.image.path
                        )
                        self.edPluginReadImageHeader = self.loadPlugin(
                            self.strPluginReadImageHeaderName)
                        self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
                        self.edPluginReadImageHeader.executeSynchronous()
                        xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
                        if xsDataResultReadImageHeader is not None:
                            edPluginLabelitIndexing = self.loadPlugin(
                                self.strIndexingLabelitPluginName)
                            xsDataInputLabelitIndexing = XSDataInputLabelitIndexing(
                            )
                            xsDataInputLabelitIndexing.image.append(
                                XSDataImage(
                                    xsDataResultControlImageQualityIndicator.
                                    image.path))
                            edPluginLabelitIndexing.setDataInput(
                                xsDataInputLabelitIndexing)
                            self.listPluginLabelit.append([
                                edPluginLabelitIndexing,
                                xsDataResultControlImageQualityIndicator
                            ])
                            edPluginLabelitIndexing.execute()
                for tupleLabelit in self.listPluginLabelit:
                    edPluginLabelitIndexing = tupleLabelit[0]
                    xsDataResultControlImageQualityIndicator = tupleLabelit[1]
                    edPluginLabelitIndexing.synchronize()
                    if not edPluginLabelitIndexing.isFailure(
                    ) and edPluginLabelitIndexing.dataOutput is not None:
                        xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput(
                        )
                        xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(
                            xsDataResultLabelitIndexing)
                        selectedSolution = xsDataIndexingResult.selectedSolution
                        if selectedSolution is not None:
                            xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
Ejemplo n.º 41
0
                makeFile.write("LOADER = %s  %s" %
                               (fortranCompiler, os.linesep))
#            elif oneline.startswith("ARCHFLAGS"):
#                makeFile.write("ARCHFLAGS = -shared -o %s" % (os.linesep))
#            elif oneline.startswith("ARCH"):
#                makeFile.write("ARCH = %s %s" % (fortranCompiler, os.linesep))
#            elif  oneline.startswith("RANLIB"):
#                makeFile.write("RANLIB = ls %s" % (os.linesep))
            elif oneline.startswith("BLASLIB"):
                makeFile.write("BLASLIB = %s %s" % (libblas, os.linesep))
            else:
                makeFile.write(oneline)

        makeFile.close()
        try:
            installBlas.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
        except Exception:
            EDVerbose.ERROR("Error for BLAS in the 'make' step")
        if not os.path.isdir(
                os.path.join(installBlas.getDestinationDirectory(), "lib")):
            os.makedirs(
                os.path.join(installBlas.getDestinationDirectory(), "lib"))
        blasPath = os.path.join(installBlas.getDestinationDirectory(), "lib")
        shutil.copyfile(
            os.path.join(installBlas.getSourceDirectory(), libblas),
            os.path.join(blasPath, libblas))

        ################################################################################
        # END of Blas / Start of Lapack
        ################################################################################
        if lapackPath is None:
Ejemplo n.º 42
0
 def semaphoreNbThreadsRelease(self):
     """Method to release the semaphore that controls the number of threads running concurrently"""
     EDUtilsParallel.semaphoreNbThreadsRelease()
Ejemplo n.º 43
0
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_3.process")
        EDUtilsParallel.initializeNbThread()
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                bDoIndexing = True
        # Loop through all the incoming reference images
        listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators(
        )
        listPlugin = []
        for xsDataImage in listXSDataImage:
            self.edPluginMXWaitFile = self.loadPlugin(
                self.strPluginMXWaitFileName)
            xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
            xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
            xsDataInputMXWaitFile.setTimeOut(
                XSDataTime(self.fMXWaitFileTimeOut))
            self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
            self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
            self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(xsDataImage.path.value):
                self.edPluginMXWaitFile = self.loadPlugin(
                    self.strPluginMXWaitFileName)
                xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
                xsDataInputMXWaitFile.setSize(XSDataInteger(5000000))
                xsDataInputMXWaitFile.setTimeOut(
                    XSDataTime(self.fMXWaitFileTimeOut))
                self.DEBUG("Wait file timeOut set to %f" %
                           self.fMXWaitFileTimeOut)
                self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(xsDataImage.path.value):
                strError = "Time-out while waiting for image %s" % xsDataImage.path.value
                self.error(strError)
                self.addErrorMessage(strError)
                self.setFailure()
            else:
                if self.bUseThinClient:
                    strPluginName = self.strPluginNameThinClient
                else:
                    strPluginName = self.strPluginName
                edPluginPluginExecImageQualityIndicator = self.loadPlugin(
                    strPluginName)
                self.listPluginExecImageQualityIndicator.append(
                    edPluginPluginExecImageQualityIndicator)
                xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength(
                )
                xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage)
                edPluginPluginExecImageQualityIndicator.setDataInput(
                    xsDataInputDistlSignalStrength)
                edPluginPluginExecImageQualityIndicator.execute()
                edPluginControlBackground3D = self.loadPlugin(
                    self.strPluginNameControlBackground3D)
                listPlugin.append([
                    edPluginPluginExecImageQualityIndicator,
                    edPluginControlBackground3D
                ])
                xsDataInputControlBackground3D = XSDataInputControlBackground3D(
                )
                xsDataInputControlBackground3D.addImage(
                    XSDataFile(xsDataImage.path))
                edPluginControlBackground3D.dataInput = xsDataInputControlBackground3D
                edPluginControlBackground3D.execute()
        listIndexing = []
        # Synchronize all image quality indicator plugins and upload to ISPyB
        xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators(
        )
        for pluginPair in listPlugin:
            edPluginPluginExecImageQualityIndicator = pluginPair[0]
            edPluginControlBackground3D = pluginPair[1]
            edPluginPluginExecImageQualityIndicator.synchronize()
            edPluginControlBackground3D.synchronize()
            xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
                                             edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
            if edPluginControlBackground3D.dataOutput.imageBackground != []:
                xsDataImageQualityIndicators.background3D_estimate = edPluginControlBackground3D.dataOutput.imageBackground[
                    0].estimate
            self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(
                xsDataImageQualityIndicators)
            xsDataISPyBImageQualityIndicators = \
                XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
            xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(
                xsDataISPyBImageQualityIndicators)
#        print xsDataInputStoreListOfImageQualityIndicators.marshal()
        if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
            self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
            self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
            self.edPluginISPyB.execute()
        #
        if bDoIndexing:
            # Find the 5 most intensive images (TIS):
            listImage = []
            # Check that we have background3D_estimate from all images:
            has_background3D_estimate = True
            for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                if imageQualityIndicators.background3D_estimate is None:
                    has_background3D_estimate = False
            if has_background3D_estimate:
                listSorted = sorted(
                    self.xsDataResultControlImageQualityIndicators.
                    imageQualityIndicators,
                    key=lambda imageQualityIndicators: imageQualityIndicators.
                    background3D_estimate.value)
            else:
                listSorted = sorted(
                    self.xsDataResultControlImageQualityIndicators.
                    imageQualityIndicators,
                    key=lambda imageQualityIndicators: imageQualityIndicators.
                    totalIntegratedSignal.value)
            for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
                if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
                    xsDataInputReadImageHeader = XSDataInputReadImageHeader()
                    xsDataInputReadImageHeader.image = XSDataFile(
                        xsDataResultControlImageQualityIndicator.image.path)
                    self.edPluginReadImageHeader = self.loadPlugin(
                        self.strPluginReadImageHeaderName)
                    self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
                    self.edPluginReadImageHeader.executeSynchronous()
                    xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
                    if xsDataResultReadImageHeader is not None:
                        xsDataSubWedge = xsDataResultReadImageHeader.subWedge
                        self.xsDataCollection = XSDataCollection()
                        self.xsDataCollection.addSubWedge(xsDataSubWedge)
                        xsDataIndexingInput = XSDataIndexingInput()
                        xsDataIndexingInput.setDataCollection(
                            self.xsDataCollection)
                        xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(
                            xsDataIndexingInput)
                        edPluginMOSFLMIndexing = self.loadPlugin(
                            self.strIndexingMOSFLMPluginName)
                        self.listPluginMOSFLM.append([
                            edPluginMOSFLMIndexing,
                            xsDataResultControlImageQualityIndicator
                        ])
                        edPluginMOSFLMIndexing.setDataInput(
                            xsDataMOSFLMIndexingInput)
                        edPluginMOSFLMIndexing.execute()
            for tupleMOSFLM in self.listPluginMOSFLM:
                edPluginMOSFLMIndexing = tupleMOSFLM[0]
                xsDataResultControlImageQualityIndicator = tupleMOSFLM[1]
                edPluginMOSFLMIndexing.synchronize()
                if not edPluginMOSFLMIndexing.isFailure():
                    xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput
                    xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(
                        xsDataMOSFLMOutput)
                    selectedSolution = xsDataIndexingResult.selectedSolution
                    if selectedSolution is not None:
                        xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
Ejemplo n.º 44
0
        EDVerbose.DEBUG("Libpthread found in %s" % pthreadPath)
        if pthreadPath is None:
            try:
                installHDF5.configure("--prefix=%s" %
                                      (installHDF5.getDestinationDirectory()))
            except Exception:
                EDVerbose.ERROR("Error in the configure step, no pthread")
        else:
            try:
                installHDF5.configure(
                    "--prefix=%s --enable-threadsafe --with-pthread=%s" %
                    (installHDF5.getDestinationDirectory(), pthreadPath))
            except Exception:
                EDVerbose.ERROR("Error in the configure step, with pthread")
        try:
            installHDF5.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
        except Exception:
            EDVerbose.ERROR("Error in the 'make' step")

        try:
            installHDF5.make("install")
        except Exception:
            EDVerbose.ERROR("Error in the 'make install' step")
        hdfPath = installHDF5.getDestinationDirectory()
        installHDF5.cleanSources()
    else:
        hdfPath = os.path.dirname(hdfPath)
    EDVerbose.DEBUG("Building H5Py with HDF5 library from %s " % (hdfPath))
    install = EDUtilsLibraryInstaller(installDir, h5pyLibrary)
    install.checkPythonVersion()
    install.dependency("numpy", "20090405-Numpy-1.3")
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.process")
        EDUtilsParallel.initializeNbThread()
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                bDoIndexing = True
        # Loop through all the incoming reference images
        listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators(
        )
        listPlugin = []
        for xsDataImage in listXSDataImage:
            self.edPluginMXWaitFile = self.loadPlugin(
                self.strPluginMXWaitFileName)
            xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
            xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
            xsDataInputMXWaitFile.setTimeOut(
                XSDataTime(self.fMXWaitFileTimeOut))
            self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
            self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
            self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(xsDataImage.path.value):
                strError = "Time-out while waiting for image %s" % xsDataImage.path.value
                self.error(strError)
                self.addErrorMessage(strError)
                self.setFailure()
            else:
                if self.bUseThinClient:
                    strPluginName = self.strPluginNameThinClient
                else:
                    strPluginName = self.strPluginName
                edPluginPluginExecImageQualityIndicator = self.loadPlugin(
                    strPluginName)
                listPlugin.append(edPluginPluginExecImageQualityIndicator)
                self.listPluginExecImageQualityIndicator.append(
                    edPluginPluginExecImageQualityIndicator)
                xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength(
                )
                xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage)
                edPluginPluginExecImageQualityIndicator.setDataInput(
                    xsDataInputDistlSignalStrength)
                edPluginPluginExecImageQualityIndicator.execute()
        listIndexing = []
        # Synchronize all image quality indicator plugins and upload to ISPyB
        xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators(
        )
        for edPluginPluginExecImageQualityIndicator in listPlugin:
            edPluginPluginExecImageQualityIndicator.synchronize()
            xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString( \
                                             edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
            self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(
                xsDataImageQualityIndicators)
            xsDataISPyBImageQualityIndicators = \
                XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
            xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(
                xsDataISPyBImageQualityIndicators)
#        print xsDataInputStoreListOfImageQualityIndicators.marshal()
        if self.bDoISPyBUpload:
            self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
            self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
            self.edPluginISPyB.execute()
        #
        if bDoIndexing:
            # Find the 5 most intensive images (TIS):
            listImage = []
            listSorted = sorted(
                self.xsDataResultControlImageQualityIndicators.
                imageQualityIndicators,
                key=
                lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value
            )
            for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
                if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
                    xsDataInputReadImageHeader = XSDataInputReadImageHeader()
                    xsDataInputReadImageHeader.image = XSDataFile(
                        xsDataResultControlImageQualityIndicator.image.path)
                    self.edPluginReadImageHeader = self.loadPlugin(
                        self.strPluginReadImageHeaderName)
                    self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
                    self.edPluginReadImageHeader.executeSynchronous()
                    xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
                    if xsDataResultReadImageHeader is not None:
                        xsDataSubWedge = xsDataResultReadImageHeader.subWedge
                        self.xsDataCollection = XSDataCollection()
                        self.xsDataCollection.addSubWedge(xsDataSubWedge)
                        xsDataIndexingInput = XSDataIndexingInput()
                        xsDataIndexingInput.setDataCollection(
                            self.xsDataCollection)
                        xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(
                            xsDataIndexingInput)
                        edPluginMOSFLMIndexing = self.loadPlugin(
                            self.strIndexingMOSFLMPluginName)
                        self.listPluginMOSFLM.append([
                            edPluginMOSFLMIndexing,
                            xsDataResultControlImageQualityIndicator
                        ])
                        edPluginMOSFLMIndexing.setDataInput(
                            xsDataMOSFLMIndexingInput)
                        edPluginMOSFLMIndexing.execute()
            for tupleMOSFLM in self.listPluginMOSFLM:
                edPluginMOSFLMIndexing = tupleMOSFLM[0]
                xsDataResultControlImageQualityIndicator = tupleMOSFLM[1]
                edPluginMOSFLMIndexing.synchronize()
                if not edPluginMOSFLMIndexing.isFailure():
                    xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput
                    xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(
                        xsDataMOSFLMOutput)
                    selectedSolution = xsDataIndexingResult.selectedSolution
                    if selectedSolution is not None:
                        xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
Ejemplo n.º 46
0
 def process(self):
     self.addTestMethod(self.unitTestMoveToTempDir)
     self.addTestMethod(self.unitTestDetectNumberOfCPUs)
     EDUtilsParallel.uninitializeNbThread()
Ejemplo n.º 47
0
        installHDF5.downloadLibrary()
        installHDF5.unZipArchive()
        pthreadPath = installHDF5.searchCLib("libpthread.so")
        EDVerbose.DEBUG("Libpthread found in %s" % pthreadPath)
        if pthreadPath is None:
            try:
                installHDF5.configure("--prefix=%s" % (installHDF5.getDestinationDirectory()))
            except:
                EDVerbose.ERROR("Error in the configure step, no pthread")
        else:
            try:
                installHDF5.configure("--prefix=%s --enable-threadsafe --with-pthread=%s" % (installHDF5.getDestinationDirectory(), pthreadPath))
            except:
                EDVerbose.ERROR("Error in the configure step, with pthread")
        try:
            installHDF5.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
        except:
            EDVerbose.ERROR("Error in the 'make' step")

        try:
            installHDF5.make("install")
        except:
            EDVerbose.ERROR("Error in the 'make install' step")
        hdfPath = installHDF5.getDestinationDirectory()
        installHDF5.cleanSources()
    else:
        hdfPath = os.path.dirname(hdfPath)
    EDVerbose.DEBUG("Building H5Py with HDF5 library from %s " % (hdfPath))
    install = EDUtilsLibraryInstaller(installDir, h5pyLibrary)
    install.checkPythonVersion()
    install.dependency("numpy", "20090405-Numpy-1.3")
Ejemplo n.º 48
0
 def process(self):
     self.addTestMethod(self.unitTestMoveToTempDir)
     self.addTestMethod(self.unitTestDetectNumberOfCPUs)
     EDUtilsParallel.uninitializeNbThread()
Ejemplo n.º 49
0
strEdnaKernel = os.path.join(pyStrEdnaHomePath, "kernel", "src")
sys.path.append(strEdnaKernel)
if not "PYTHONPATH" in os.environ:
    os.environ["PYTHONPATH"] = strEdnaKernel
else:
    os.environ["PYTHONPATH"] += strEdnaKernel
os.environ["PYTHONPATH"] += os.pathsep + os.path.join(pyStrEdnaHomePath, "kernel", "tests", "src")
os.environ["PYTHONPATH"] += os.pathsep + os.path.join(pyStrEdnaHomePath, "kernel", "tests", "testsuite")


from EDJob import EDJob
from EDUtilsParallel import EDUtilsParallel
from EDVerbose import EDVerbose
from EDFactoryPluginStatic import EDFactoryPluginStatic

EDUtilsParallel.initializeNbThread()
EDFactoryPluginStatic.loadModule("XSDataDocumentation")
from XSDataDocumentation import XSDataString, XSDataFile, XSDataInputEpydoc, XSDataInteger

dictJobs = {}


def findPlugins(EDNAHome, EDPluginPrefix="EDPlugin", XSDataPrefix="XSData", pythonExtension=".py"):
    """
    This function is the walker that goes through all directories in EDNA_HOME directory and searches for EDNA plugins ...
    
    @param EDNAHome: the path of EDNA_HOME
    @type EDNAHome: string
    @param EDPluginPrefix: the start of the name of an EDNA Plugin
    @type EDPluginPrefix: string
    @param pythonExtension: the extension of an EDNA plugin, usually .py