Exemplo n.º 1
0
    def __init__(self, _strPluginName, _functXMLin, \
                  _functXMLout=None, _functXMLerr=None, \
                  _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
        """
        This is the constructor of the edna plugin launcher.
        
        @param _strPluginName: the name of the ENDA plugin
        @type  _strPluginName: python string
        
        @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. 
        @type  _functXMLin: python function
        
        @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
        @type  _functXMLOut: python function
         
        @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
        @type  _functXMLErr: python function 
        
        @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores  will be auto-detected. 
        @type  _iNbThreads: python integer
        
        @param _fDelay: The delay in seconds between two directories analysis 
        @type  _fDelay: python float
        
        @param _bVerbose:  Do you want the EDNA plugin execution to be verbose ?
        @type  _bVerbose: boolean

        @param _bDebug:  Do you want EDNA plugin execution debug output (OBS! very verbose) ?
        @type  _bDebug: boolean
        """
        EDLogging.__init__(self)
        self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
        EDUtilsParallel.initializeNbThread(self.__iNbThreads)
        ################################################################################
        # #We are not using the one from EDUtilsParallel to leave it to control the number of  execPlugins.
        ################################################################################
        self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
        self.__strPluginName = _strPluginName
        self.__functXMLin = _functXMLin
        self.__functXMLout = _functXMLout
        self.__functXMLerr = _functXMLerr
        self.__strCurrWorkDir = os.getcwd()
        self.__strTempDir = None
        self.__listInputPaths = []
        self.__dictCurrentlyRunning = {}
        if _bVerbose is not None:
            if _bVerbose:
                self.setVerboseDebugOn()
            else:
                self.setVerboseOff()
        if _bDebug is not None:
            if _bDebug:
                self.setVerboseDebugOn()
            else:
                self.setVerboseDebugOff()
        self.__fDelay = _fDelay  #default delay between two directory checks.
        self.__bQuit = False  # To check if we should quit the application
        self.__bIsFirstExecute = True
        signal.signal(signal.SIGTERM, self.handleKill)
        signal.signal(signal.SIGINT, self.handleKill)
Exemplo n.º 2
0
 def __init__(self):
     """
     """
     EDPluginControl.__init__(self)
     self.setXSDataInputClass(XSDataInputAlignStack)
     self.iFrames = []
     self.npArrays = []
     self.xsdMeasureOffset = None
     self.hdf5ExtraAttributes = None
     self.xsdHDF5File = None
     self.xsdHDF5Internal = None
     self.bAlwaysMOvsRef = True
     self.bDoAlign = True
     self.semAccumulator = Semaphore()
     self.semMeasure = Semaphore()
     self.semShift = Semaphore()
     self.lstSem = [
         self.locked(), self.semAccumulator, self.semMeasure, self.semShift
     ]
     self.queue = Queue()
     self.__strControlledPluginAccumulator = "EDPluginAccumulatorv1_0"
     self.__strControlledPluginMeasureFFT = "EDPluginExecMeasureOffsetv1_0"
     self.__strControlledPluginMeasureSIFT = "EDPluginExecMeasureOffsetv2_0"
     self.__strControlledPluginShift = "EDPluginExecShiftImagev1_1"
     self.__strControlledPluginHDF5 = "EDPluginHDF5StackImagesv10"
Exemplo n.º 3
0
    def parse(self):
        """
        parse options from command line
        """
        parser = optparse.OptionParser()
        parser.add_option("-V", "--version", dest="version", action="store_true",
                          help="print version of the program and quit", metavar="FILE", default=False)
        parser.add_option("-v", "--verbose",
                          action="store_true", dest="debug", default=False,
                          help="switch to debug/verbose mode")
        parser.add_option("-m", "--mask", dest="mask",
                      help="file containing the mask (for image reconstruction)", default=None)
        parser.add_option("-M", "--mode", dest="mode",
                      help="Mode can be online/offline/all", default="offline")
        parser.add_option("-o", "--out", dest="output",
                      help="file for log", default=None)
        parser.add_option("-w", "--wavelength", dest="wavelength", type="float",
                      help="wavelength of the X-Ray beam in Angstrom", default=None)
        parser.add_option("-e", "--energy", dest="energy", type="float",
                      help="energy of the X-Ray beam in keV (hc=%skeV.A)" % self.hc, default=None)
        parser.add_option("-t", "--template", dest="template", type="str",
                      help="template XML file", default=None)
        parser.add_option("-n", "--nbcpu", dest="nbcpu", type="int",
                      help="template XML file", default=self.nbcpu)


        (options, args) = parser.parse_args()

        # Analyse aruments and options
        if options.version:
            print("BioSaxs Azimuthal integration version %s" % __version__)
            sys.exit(0)
        if options.debug:
            EDVerbose.setVerboseDebugOn()
            self.debug = True
        if options.output:
            EDVerbose.setLogFileName(options.output)
        if options.mask and os.path.isfile(options.mask):
            self.maskfile = options.mask
        if options.template and os.path.isfile(options.template):
            self.xml = open(options.template).read()
        if options.wavelength:
            self.wavelength = 1e-10 * options.wavelength
        elif options.energy:
            self.wavelength = 1e-10 * self.hc / options.energy
        if options.mode=="offline":
            self.mode = "offline"
            self.newerOnly = False
        elif options.mode=="online":
            self.mode = "dirwarch"
            self.newerOnly = True
        elif options.mode=="dirwatch":
            self.mode = "dirwarch"
            self.newerOnly = False
        self.cpu_sem = Semaphore(options.nbcpu)
        self.nbcpu = options.nbcpu
        self.dataFiles = [f for f in args if os.path.isfile(f)]
        if not self.dataFiles:
            raise RuntimeError("Please provide datafiles or read the --help")
Exemplo n.º 4
0
 def __init__(self):
     """
     Constructor of the main pure virtual class.
     This constructor implements:
     - the creation of the semaphore
     - definition of timer object (uninitialized as potentially not used)
     """
     object.__init__(self)
     with self.__class__.__semaphoreId:
         self.__class__.__iId_class += 1
         self.__iId = self.__class__.__iId_class
     self.__semaphore = Semaphore()
     self.__fTimeInit = None
     self.__fTimeEnd = None
     self.__classname = None
Exemplo n.º 5
0
 def __init__(self):
     object.__init__(self)
     self.__pySemaphoreWrite = Semaphore()
     self.__bIsVerbose = True
     self.__bIsVerboseDebug = False
     self.__bIsTest = False
     self.__edLogFile = EDLogFile()
Exemplo n.º 6
0
 def __init__(self, runId, first_curve=None):
     self.id = runId
     self.buffer = None
     self.first_curve = first_curve
     self.frames = {}  #key: id, value: HPLCframe instance
     self.curves = []
     self.for_buffer = []
     self.hdf5_filename = None
     self.hdf5 = None
     self.chunk_size = 250
     self.lock = Semaphore()
     if first_curve:
         self.files.append(first_curve)
     self.max_size = None
     self.time = None
     self.gnom = None
     self.Dmax = None
     self.total = None
     self.volume = None
     self.Rg = None
     self.Rg_Stdev = None
     self.I0 = None
     self.I0_Stdev = None
     self.quality = None
     self.q = None
     self.size = None
Exemplo n.º 7
0
    def __init__(self, _strPluginName, _functXMLin, \
                  _functXMLout=None, _functXMLerr=None, \
                  _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
        """
        This is the constructor of the edna plugin launcher.
        
        @param _strPluginName: the name of the ENDA plugin
        @type  _strPluginName: python string
        
        @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. 
        @type  _functXMLin: python function
        
        @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
        @type  _functXMLOut: python function
         
        @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
        @type  _functXMLErr: python function 
        
        @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores  will be auto-detected. 
        @type  _iNbThreads: python integer
        
        @param _fDelay: The delay in seconds between two directories analysis 
        @type  _fDelay: python float
        
        @param _bVerbose:  Do you want the EDNA plugin execution to be verbose ?
        @type  _bVerbose: boolean

        @param _bDebug:  Do you want EDNA plugin execution debug output (OBS! very verbose) ?
        @type  _bDebug: boolean
        """
        EDLogging.__init__(self)
        self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
        EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of  execPlugins.
################################################################################
        self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
        self.__strPluginName = _strPluginName
        self.__functXMLin = _functXMLin
        self.__functXMLout = _functXMLout
        self.__functXMLerr = _functXMLerr
        self.__strCurrWorkDir = os.getcwd()
        self.__strTempDir = None
        self.__listInputPaths = []
        self.__dictCurrentlyRunning = {}
        if _bVerbose is not None:
            if _bVerbose:
                self.setVerboseDebugOn()
            else:
                self.setVerboseOff()
        if _bDebug is not None:
            if _bDebug:
                self.setVerboseDebugOn()
            else:
                self.setVerboseDebugOff()
        self.__fDelay = _fDelay #default delay between two directory checks.
        self.__bQuit = False    # To check if we should quit the application
        self.__bIsFirstExecute = True
        signal.signal(signal.SIGTERM, self.handleKill)
        signal.signal(signal.SIGINT, self.handleKill)
Exemplo n.º 8
0
    def __init__(self, strPluginName, iNbCpu=None):
        EDLogging.__init__(self)
        self.pluginName = strPluginName
        self.startTime = time.time()
        try:
            self.iNbCpu = int(iNbCpu)
        except:
            self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs()

        self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu)
        self.__semaphoreNbThreads = Semaphore(self.iNbCpu)
        EDUtilsParallel.initializeNbThread(self.iNbCpu)
        self.jobQueue = Queue()
        self.processingSem = Semaphore()
        self.statLock = Semaphore()
        self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
        self.lastFailure = "No job Failed (yet)"
        self.lastSuccess = "No job succeeded (yet)"
Exemplo n.º 9
0
    def __init__(self):
        self.XML = "<XSDataInput>\
<normalizedImage><path><value>${FULLPATH}</value></path></normalizedImage>\
<correctedImage><path><value>${DIRDIRNAME}/2d/${BASENAME}.edf</value></path></correctedImage>\
<normalizedImageSize><value>4100000</value></normalizedImageSize>\
<integratedCurve><path><value>${DIRDIRNAME}/1d/${BASENAME}.edf</value></path></integratedCurve>\
<maskFile><path><value>${MASKFILE}</value></path></maskFile>\
<code><value>BSA</value></code>\
</XSDataInput>"
        self.maskfile = None
        self.dataFiles = []
        self.wavelength = 1.0
        self.debug = False
        self.mode = "offline"
        self.newerOnly = False
        self.nbcpu = multiprocessing.cpu_count()
        self.cpu_sem = Semaphore(self.nbcpu)
        self.process_sem = Semaphore()
        self.queue = Queue()
Exemplo n.º 10
0
 def __init__(self):
     """
     Constructor of the main pure virtual class.
     This constructor implements:
     - the creation of the semaphore
     - definition of timer object (uninitialized as potentially not used)
     """
     object.__init__(self)
     with self.__class__.__semaphoreId:
         self.__class__.__iId_class += 1
         self.__iId = self.__class__.__iId_class
     self.__semaphore = Semaphore()
     self.__fTimeInit = None
     self.__fTimeEnd = None
     self.__classname = None
Exemplo n.º 11
0
 def initializeNbThread(cls, _iNbThread=None):
     """
     Class method:
     Initialises a semaphore with the right number of threads
     @param _iNbThread: the maximum number of threads concurrently running and CPU intensive
     @type _iNbThread: integer
     """
     with cls._semaphoreInit:
         if cls._semaphoreNbThreads is None:
             if _iNbThread is None:
                 _iNbThread = cls.detectNumberOfCPUs()
             EDVerbose.DEBUG(
                 "Initializing EDUtilsParallel semaphoreNbThreads to %s" %
                 _iNbThread)
             cls._iNbThreads = _iNbThread
             cls._semaphoreNbThreads = Semaphore(_iNbThread)
         else:
             if cls._iNbThreads != _iNbThread:
                 EDVerbose.WARNING(
                     "cls._semaphoreNbThreads was already initialized to %s, not changing to %s"
                     % (cls._iNbThreads, _iNbThread))
Exemplo n.º 12
0
class EDObject(object):
    """
    Virtual base class for all EDNA Objects (classes).
    It offers some synchronization and locking capabilities to make the code thread safe.
    """
    __semaphoreId = Semaphore()
    __iId_class = 0

    def __init__(self):
        """
        Constructor of the main pure virtual class.
        This constructor implements:
        - the creation of the semaphore
        - definition of timer object (uninitialized as potentially not used)
        """
        object.__init__(self)
        with self.__class__.__semaphoreId:
            self.__class__.__iId_class += 1
            self.__iId = self.__class__.__iId_class
        self.__semaphore = Semaphore()
        self.__fTimeInit = None
        self.__fTimeEnd = None
        self.__classname = None


    def getId(self):
        return self.__iId


    def getClassName(self):
        """
        Retrieves the name of the class
        @return: the name of the class 
        @rtype: string 
        """
        return self.__class__.__name__


    def synchronizeOn(self):
        """
        This method must be used in together with the method synchronizeOff().
        This method makes the code threadsafe till the method synchronizeOff
        is called.
        """
        self.__semaphore.acquire()


    def synchronizeOff(self):
        """
        This method must be used in together with the method synchronizeOn().
        """
        self.__semaphore.release()


    def locked(self):
        return self.__semaphore


    def setTimeInit(self):
        """
        Initializes the timer for the object
        """
        if self.__fTimeInit is None:
            self.__fTimeInit = time.time()




    def getTimeInit(self):
        """
        Retrieves the time of initialization
        @return: number of seconds since epoch
        @rtype: float  
        """
        return self.__fTimeInit


    def setTimeEnd(self):
        """
        Set the end of calculation time for the given object
        """
        if self.__fTimeEnd is None:
            self.__fTimeEnd = time.time()


    def getTimeEnd(self):
        """
        Retrieves the time of end of task
        @return: number of seconds since epoch
        @rtype: float  
        """
        return self.__fTimeEnd

    def getRunTime(self):
        """
        @returns: the RunTime for the given object
        @rtype: float
        """
        fRetrunRunTime = 0.0
        if self.__fTimeInit is not None:
            if self.__fTimeEnd is None:
                fRetrunRunTime = time.time() - self.__fTimeInit
            else:
                fRetrunRunTime = self.__fTimeEnd - self.__fTimeInit
        return fRetrunRunTime
Exemplo n.º 13
0
class EDParallelExecute(EDLogging):
    """ 
    A class helping to make a multi-threaded application from a plugin name and a list of files. 
    """

    def __init__(self, _strPluginName, _functXMLin, \
                  _functXMLout=None, _functXMLerr=None, \
                  _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
        """
        This is the constructor of the edna plugin launcher.
        
        @param _strPluginName: the name of the ENDA plugin
        @type  _strPluginName: python string
        
        @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. 
        @type  _functXMLin: python function
        
        @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
        @type  _functXMLOut: python function
         
        @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
        @type  _functXMLErr: python function 
        
        @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores  will be auto-detected. 
        @type  _iNbThreads: python integer
        
        @param _fDelay: The delay in seconds between two directories analysis 
        @type  _fDelay: python float
        
        @param _bVerbose:  Do you want the EDNA plugin execution to be verbose ?
        @type  _bVerbose: boolean

        @param _bDebug:  Do you want EDNA plugin execution debug output (OBS! very verbose) ?
        @type  _bDebug: boolean
        """
        EDLogging.__init__(self)
        self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
        EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of  execPlugins.
################################################################################
        self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
        self.__strPluginName = _strPluginName
        self.__functXMLin = _functXMLin
        self.__functXMLout = _functXMLout
        self.__functXMLerr = _functXMLerr
        self.__strCurrWorkDir = os.getcwd()
        self.__strTempDir = None
        self.__listInputPaths = []
        self.__dictCurrentlyRunning = {}
        if _bVerbose is not None:
            if _bVerbose:
                self.setVerboseDebugOn()
            else:
                self.setVerboseOff()
        if _bDebug is not None:
            if _bDebug:
                self.setVerboseDebugOn()
            else:
                self.setVerboseDebugOff()
        self.__fDelay = _fDelay #default delay between two directory checks.
        self.__bQuit = False    # To check if we should quit the application
        self.__bIsFirstExecute = True
        signal.signal(signal.SIGTERM, self.handleKill)
        signal.signal(signal.SIGINT, self.handleKill)


    def runEDNA(self, _pyListInputPaths=[ "." ], _strMode="dirwatch", _bNewerOnly=False):
        """
        This method runs the parallel execution on the list of directories.

        @param _pyListInputPaths: the name of the directories to look after.
        @type  _pyListInputPaths: python list of strings
        
        @param _strMode: can be dirwatch, inotify, or OffLine (inotify being not yet implemented)
        @type _strMode: python string

        @param _bNewerOnly: in online mode, process only new files (appearing after the program has started), by default it will process all files then wait for newer files and process them.
        @type _bNewerOnly: boolean
        """
        self.moveToTempDir()
        self.__listInputPaths = _pyListInputPaths
        if _strMode == "dirwatch":
            self.watch_directories(_bNewerOnly)
        elif _strMode == "inotify":
            print("inotify online notify mode not yet implemented")
            raise
        else: #mode offline
            self.runEdnaFunction(self.__listInputPaths, _bIncludeSubdirs=True)
            self.waitForAllProcessToFinish()


    def moveToTempDir(self):
        """
        Create a temporary directory and put all logs there
        """
        self.__strCurrWorkDir = os.getcwd()
        self.__strTempDir = tempfile.mkdtemp(suffix='.log', prefix='edna-')
        self.screen("The log directory of EDNA will be in " + self.__strTempDir)
        os.chdir(self.__strTempDir)


    def start(self, _strXmlInput):
        """
        Launch EDNA with the given XML stream
        @param _strXmlInput:  XML to be passed to the plugin
        @type  _strXmlInput: python string representing the XML data structure
        """
        jobid = None
        if _strXmlInput not in ["", None]:
            job = EDJob(self.__strPluginName)
            job.setDataInput(_strXmlInput)
            job.connectFAILURE(self.failureJobExecution)
            job.connectSUCCESS(self.successJobExecution)
            job.connectCallBack(self.unregisterJob)
            self.semaphoreNbThreadsAcquire()
            jobid = job.execute()
            self.DEBUG("Running Job id %s" % jobid)
            if jobid is None:
                self.semaphoreNbThreadsRelease()
        return jobid


    def successJobExecution(self, _jobId):
        """
        Method called when the execution of the plugin finishes with success

        @param  _jobId: string of type EDPluginName-number 
        """
        self.DEBUG("EDParallelExcecute.successJobExecution for %s" % _jobId)
        self.semaphoreNbThreadsRelease()
        with self.locked():
            if self.__functXMLout is not None:
                job = EDJob.getJobFromID(_jobId)
                self.__functXMLout(job.getPlugin().getDataInput(), job.getPlugin().getDataOutput())


    def failureJobExecution(self, _jobId):
        """
        Method called when the execution of the plugin finishes with failure 
        
        @param  _jobId: string of type EDPluginName-number
        """
        self.DEBUG("EDParallelExcecute.failureJobExecution for %s" % _jobId)
        self.semaphoreNbThreadsRelease()
        with self.locked():
            if self.__functXMLerr is not None:
                self.__functXMLerr(EDJob.getJobFromID(_jobId).getPlugin().getDataInput())


    def unregisterJob(self, _jobid):
        """
        remove the filename from the list of files currently under processing

        @param  _jobId: string of type EDPluginName-number          
        """
        with self.locked():
            for oneKey in self.__dictCurrentlyRunning.copy():
                if self.__dictCurrentlyRunning[oneKey] == _jobid:
                    self.__dictCurrentlyRunning.pop(oneKey)


    def runEdnaFunction(self, _listNewFiles, _bIncludeSubdirs=False):
        """
        This method is the launcher for new files found by watch_directories ; it is also called directly in offline mode.
        
        @param  _listNewFiles: list of files newly created in the directory.
        @type   _listNewFiles: python list of strings.
        @param  _bIncludeSubdirs: should we include sub-directories ? yes for offline and no for online.
        @type   _bIncludeSubdirs: boolean
        """
        for oneFile in _listNewFiles:
            if os.path.isdir(oneFile) and _bIncludeSubdirs == True:
                for root, _, onesubdirfiles in os.walk(oneFile):
                    for onesubdirfile in onesubdirfiles:
                        strFilename = os.path.abspath(os.path.join(root, onesubdirfile))
                        if self.__bQuit == True:
                            return
                        self.processOneFile(strFilename)
            elif os.path.isfile(oneFile):
                if self.__bQuit == True:
                    return
                self.processOneFile(oneFile)


    def processOneFile(self, _strFilename):
        """
        Process on file by calling subsequently the XML generator and the start method unless this file
        is already under process (can happend with the watch_directory method).

        @param  _strFilename: filename to process
        @type _strFilename: string
        """
        if _strFilename not in self.__dictCurrentlyRunning:
            with self.locked():
                self.__dictCurrentlyRunning[_strFilename] = self.__strPluginName
            strXmlData = self.__functXMLin(_strFilename)
            if strXmlData in [None, ""]:
                self.log("Not processing % s" % _strFilename)
                with self.locked():
                    self.__dictCurrentlyRunning.pop(_strFilename)
            else:
                self.screen("Processing % s" % _strFilename)
                jobid = self.start(strXmlData)
                with self.locked():
                    if jobid is None:
                        self.__dictCurrentlyRunning.pop(_strFilename)
                    else:
                        self.__dictCurrentlyRunning[_strFilename] = jobid

    def watch_directories (self, _bNewerOnly=False):
        """
        Continuously monitors the paths and their subdirectories
        for changes.  If any files or directories are modified,
        the callable function ( here the method self.runEdnaFunction() ) is called 
        with a list of the modified paths of both
        files and directories.  This function can return a Boolean value
        for rescanning; if it returns True, the directory tree will be
        rescanned without calling the function for any found changes.
        (This is so this function can write changes into the tree and prevent itself
        from being immediately called again.)

        @param _bNewerOnly : Do you want to process only newer files  
        @type _bNewerOnly  : Boolean

        Basic principle: pyDictAllFiles is a dictionary mapping paths to
        modification times.  We repeatedly crawl through the directory
        tree rooted at 'path', doing a stat() on each file and comparing
        the modification time.

        """
        dictAllFiles = {}
        dictRemainingFiles = {}
        listChangedFiles = []

        def internalUpdateDict (unused, dirname, files):
            "Traversal function for directories"
            for strFilename in files:
                path = os.path.join(dirname, strFilename)

                try:
                    tempStat = os.stat(path)
                except os.error:
                    # If a file has been deleted between os.path.walk()
                    # scanning the directory and now, we'll get an
                    # os.error here.  Just ignore it -- we'll report
                    # the deletion on the next pass through the main loop.
                    continue

                mtime = dictRemainingFiles.get(path)
                if mtime is not None:
                    # Record this file as having been seen
                    del dictRemainingFiles[path]
                    # File's mtime has been changed since we last looked at it.
                    if tempStat.st_mtime > mtime:
                        listChangedFiles.append(path)
                else:
                    # No recorded modification time, so it must be
                    # a brand new file.
                    listChangedFiles.append(path)

                # Record current mtime of file.
                dictAllFiles[path] = tempStat.st_mtime

        if _bNewerOnly:
            for path in self.__listInputPaths:
                os.path.walk(path, internalUpdateDict, None)

        # Main loop
        rescan = False
        while not self.__bQuit:
            listChangedFiles = []
            dictRemainingFiles = dictAllFiles.copy()
            dictAllFiles = {}
            for path in  self.__listInputPaths:
                os.path.walk(path, internalUpdateDict, None)
            #removed_list = dictRemainingFiles.keys()
            if rescan:
                rescan = False
            elif listChangedFiles:
                rescan = self.runEdnaFunction(listChangedFiles, _bIncludeSubdirs=False)
            time.sleep(self.__fDelay)
        print("Quitting the online mode.")


    def handleKill(self, signum, frame):
        """ 
        This method is launched when the program catches ctrl-c or get killed. It initialize the exit of the program
        """
        self.__bQuit = True
        sys.stderr.write("Exit requested by signal %s with frame %s.\n" % (signum, frame))
        self.waitForAllProcessToFinish()
        os.chdir(self.__strCurrWorkDir)


    def flush(self):
        """
        This method calls the functXMLin a few times with a flush=True argument or without arguments and finishes the work 
        """
        bFinished = False
        while not bFinished:
            xml = None

            try:
                xml = self.__functXMLin(None, flush=True)
            except TypeError:
                try:
                    xml = self.__functXMLin("", flush=True)
                except TypeError:
                    try:
                        xml = self.__functXMLin("")
                    except TypeError:
                        try:
                            xml = self.__functXMLin("")
                        except TypeError:
                            xml = None
        if (xml is None) or (xml == ""):
            bFinished = True
        else:
            self.screen ("Flushing data ...")
            self.start(xml)


    def waitForAllProcessToFinish(self):
        """
        as it names says, this method waits for all plug-ins which are currently running to finish before returning.
        """
        self.screen("Waiting for launched jobs to finish .")
        while (self.getNbRunning() > 0):
            time.sleep(1)
            sys.stderr.write(".")
        sys.stderr.write("Done.\n")
        EDJob.stats()


    def cleanUp(self, listMethods=[]):
        """
        Final hook if you need to execute something after all processes finished (like killAllWorkers in SPD) 
        @param listMethods: allows to finish some things in the plugin. 
        @type listMethods: list of strings representing names of methods of the plugin to be called.
        """
        self.waitForAllProcessToFinish()
        for strOneMethod in  listMethods:
            try:
                print("calling edPlugin.%s" % strOneMethod)
                exec("edPlugin.%s" % strOneMethod)
            except Exception:
                print("error in processing %s" % strOneMethod)


################################################################################
# Nota: there are 2 levels of controls for the number of thread currently running:
# * One here to limit the number of control plugin running at once
# * One on the Exec plugin level for finer grain optimisation 
################################################################################
    def semaphoreNbThreadsAcquire(self):
        """Method to acquire the semaphore that controls the number of plugins running concurrently"""
#        pass
        self.__semaphoreNbThreads.acquire()


    def semaphoreNbThreadsRelease(self):
        """Method to release the semaphore that controls the number of plugins running concurrently"""
#        pass
        self.__semaphoreNbThreads.release()


    def getNbRunning(self):
        """
        Class method:
        getter for the number of CPU-active threads running
    
        @return: the number of CPU-active threads runnings
        @rtype: integer
        """
        #return EDUtilsParallel.getNbRunning()
        return self.__iNbThreads - self.__semaphoreNbThreads._Semaphore__value
Exemplo n.º 14
0
class Reprocess(EDLogging):
    def __init__(self, strPluginName, iNbCpu=None):
        EDLogging.__init__(self)
        self.pluginName = strPluginName
        self.startTime = time.time()
        try:
            self.iNbCpu = int(iNbCpu)
        except:
            self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs()

        self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu)
        self.__semaphoreNbThreads = Semaphore(self.iNbCpu)
        EDUtilsParallel.initializeNbThread(self.iNbCpu)
        self.jobQueue = Queue()
        self.processingSem = Semaphore()
        self.statLock = Semaphore()
        self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
        self.lastFailure = "No job Failed (yet)"
        self.lastSuccess = "No job succeeded (yet)"


    def startJob(self, xsd):
        """
        @param xsd: XML data structure as a string or path to a string
        @return: jobID which is a sting: Plugin-000001
        """
        self.DEBUG("In %s.startJob()" % self.__class__.__name__)
        if type(xsd) in types.StringTypes:
            if xsd.strip() == "":
                return
            if os.path.isfile(xsd):
                xsd = open(xsd, "rb").read()
        edJob = EDJob(self.pluginName)
        if edJob is None:
            return "Error in load Plugin"
        jobId = edJob.getJobId()
        edJob.setDataInput(xsd)
        self.jobQueue.put(edJob)
        if self.processingSem._Semaphore__value > 0 :
            t = threading.Thread(target=self.startProcessing)
            t.start()
        return jobId

    def startProcessing(self):
        """
        Process all jobs in the queue.
        """
        with self.processingSem:
            while not self.jobQueue.empty():
                self.__semaphoreNbThreads.acquire()
                edJob = self.jobQueue.get()
                edJob.connectSUCCESS(self.successJobExecution)
                edJob.connectFAILURE(self.failureJobExecution)
                edJob.execute()
#                edJob.synchronize()

    def successJobExecution(self, jobId):
        self.DEBUG("In %s.successJobExecution(%s)" % (self.__class__.__name__, jobId))
        with self.locked():
            self.__semaphoreNbThreads.release()
            EDJob.cleanJobfromID(jobId, False)
            self.lastSuccess = jobId
            gc.collect()

    def failureJobExecution(self, jobId):
        self.DEBUG("In %s.failureJobExecution(%s)" % (self.__class__.__name__, jobId))
        with self.locked():
            self.__semaphoreNbThreads.release()
            EDJob.cleanJobfromID(jobId, False)
            self.lastFailure = jobId
            sys.stdout.flush()
            sys.stderr.flush()
            gc.collect()

    def getRunning(self):
        """
        retrieve the list of plugins currently under execution (with their plugin-Id)
        """
        return EDStatus.getRunning()

    def getSuccess(self):
        """
        retrieve the list of plugins finished with success (with their plugin-Id)
        """
        return EDStatus.getSuccess()

    def getFailure(self):
        """
        retrieve the list of plugins finished with failure (with their plugin-Id)
        """
        return EDStatus.getFailure()

    def collectStatistics(self):
        """
        Retrieve some statistics on all EDNA-Jobs
        @return: a page of information about EDNA-jobs
        """
        t = threading.Thread(target=self.statistics)
        t.start()


    def statistics(self):
        """
        retrieve some statistics about past jobs.
        """
        with self.statLock:
            fStartStat = time.time()
            self.lastStatistics = EDJob.stats()
            self.lastStatistics += os.linesep + "Statistics collected on %s, the collect took: %.3fs" % (time.asctime(), time.time() - fStartStat)
        return self.lastStatistics

    def getStatistics(self):
        """
        just return statistics previously calculated
        """
        return self.lastStatistics

    def getJobOutput(self, jobId):
        """
        Retrieve XML output form a job
        @param jobId: name of the job
        @return: output from a job
        """
        return EDJob.getDataOutputFromId(jobId)

    def getJobInput(self, jobId):
        """
        Retrieve XML input from a job
        @param jobId: name of the job
        @return: xml input from a job
        """
        return EDJob.getDataInputFromId(jobId)

    def join(self):
        """
        wait for all jobs to finish
        """
        while not (self.jobQueue.empty() and \
                (self.__semaphoreNbThreads._Semaphore__value == self.iNbCpu) and \
                (EDUtilsParallel.getNbRunning() == 0) and \
                (self.processingSem._Semaphore__value == 1) and\
                (len(EDStatus.getRunning()) == 0)):
            time.sleep(1)
class EDPluginBioSaxsProcessOneFilev1_4(EDPluginControl):
    """
    Control plugin that does the same as previously without sub-plugin call ...
    except WaitFile which is still called.

    Nota normalization is done AFTER integration not before as previously
    
    New in version 1.4: return I/Q/stderr via XSDataArrays
     
    """
    cpWaitFile = "EDPluginWaitFile"
    integrator = pyFAI.AzimuthalIntegrator()
    integrator.wavelength = 1e-10
    CONF_DUMMY_PIXEL_VALUE = "DummyPixelValue"
    CONF_DUMMY_PIXEL_DELTA = "DummyPixelDelta"
    CONF_OPENCL_DEVICE = "DeviceType"
    __configured = False
    dummy = -2
    delta_dummy = 1.1
    semaphore = Semaphore()
    maskfile = None
    if pyFAI.opencl.ocl is None:
        METHOD = "lut"
    else:
        METHOD = "lut_ocl_gpu"

    def __init__(self):
        """
        """
        EDPluginControl.__init__(self)
        self.setXSDataInputClass(XSDataInputBioSaxsProcessOneFilev1_0)
        self.__edPluginWaitFile = None
        self.rawImage = None
        self.rawImageSize = XSDataInteger(1024)
        self.normalizedImage = None
        self.integratedCurve = None
        self.integratedImage = None
        self.lstExecutiveSummary = []
        self.sample = None
        self.experimentSetup = None
        self.integrator_config = {}
        self.scale = None
        self.detector = None
        self.xsDataResult = XSDataResultBioSaxsProcessOneFilev1_0()


    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.checkParameters")
        self.checkMandatoryParameters(self.dataInput, "Data Input is None")
        self.checkMandatoryParameters(self.dataInput.rawImage, "No raw image provided")
        self.checkMandatoryParameters(self.dataInput.sample, "No sample information provided")
        self.checkMandatoryParameters(self.dataInput.experimentSetup, "No experimental setup provided")
        self.checkMandatoryParameters(self.dataInput.integratedCurve, "Missing integratedCurve")

    def configure(self):
        """
        Configures the plugin from the configuration file with the following parameters:
         - DummyPixelValue: the value be assigned to dummy pixels.
         - DummyPixelDelta: the value be assigned to delta dummy.
         - DeviceType: "lut_ocl_1,3" will select device #3 on first platform #1 
        """
        EDPluginControl.configure(self)
        if not self.__configured:
            with self.semaphore:
                if not self.__configured:
                    self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.configure")
                    dummy = self.config.get(self.CONF_DUMMY_PIXEL_VALUE)
                    if dummy is None:
                        strMessage = 'EDPluginBioSaxsProcessOneFilev1_4.configure: %s Configuration parameter missing: \
            %s, defaulting to "%s"' % (self.getBaseName(), self.CONF_DUMMY_PIXEL_VALUE, self.dummy)
                        self.WARNING(strMessage)
                        self.addErrorWarningMessagesToExecutiveSummary(strMessage)
                    else:
                        self.__class__.dummy = float(dummy)
                    ddummy = self.config.get(self.CONF_DUMMY_PIXEL_DELTA)
                    if ddummy is None:
                        strMessage = 'EDPluginBioSaxsProcessOneFilev1_4.configure: %s Configuration parameter missing: \
            %s, defaulting to "%s"' % (self.getBaseName(), self.CONF_DUMMY_PIXEL_DELTA, self.delta_dummy)
                        self.WARNING(strMessage)
                        self.addErrorWarningMessagesToExecutiveSummary(strMessage)
                    else:
                        self.__class__.delta_dummy = float(ddummy)
                    method = self.config.get(self.CONF_OPENCL_DEVICE)
                    if method is None:
                        strMessage = 'EDPluginBioSaxsProcessOneFilev1_4.configure: %s Configuration parameter missing: \
            %s, defaulting to "%s"' % (self.getBaseName(), self.CONF_OPENCL_DEVICE, self.METHOD)
                        self.WARNING(strMessage)
                        self.addErrorWarningMessagesToExecutiveSummary(strMessage)
                    else:
                        self.__class__.METHOD = method
                    self.__class__.__configured = True


    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.preProcess")
        self.__edPluginWaitFile = self.loadPlugin(self.cpWaitFile)
        if self.dataInput.rawImageSize is not None:
            self.rawImageSize = self.dataInput.rawImageSize
        if self.dataInput.rawImage is not None:
            self.rawImage = self.dataInput.rawImage.path.value
        self.integratedCurve = self.dataInput.integratedCurve.path.value
        curveDir = os.path.dirname(self.integratedCurve)
        if not os.path.isdir(curveDir):
            try:
                os.mkdir(curveDir)
            except OSError:
                # could occur in race condition ...
                pass

        self.sample = self.dataInput.sample
        self.experimentSetup = self.dataInput.experimentSetup
        self.detector = self.experimentSetup.detector.value
        if self.detector.lower() == "pilatus":
            self.detector = "Pilatus1M"
        else:
            self.detector = self.detector.capitalize()
        self.integrator_config = {'dist': self.experimentSetup.detectorDistance.value,
                                  'pixel1': self.experimentSetup.pixelSize_2.value, # flip X,Y
                                  'pixel2': self.experimentSetup.pixelSize_1.value, # flip X,Y
                                  'poni1': self.experimentSetup.beamCenter_2.value * self.experimentSetup.pixelSize_2.value,
                                  'poni2': self.experimentSetup.beamCenter_1.value * self.experimentSetup.pixelSize_1.value,
                                  'rot1': 0.0,
                                  'rot2': 0.0,
                                  'rot3': 0.0,
                                  'splineFile': None,
                                  'detector': self.detector
                                  }
        i0 = self.experimentSetup.beamStopDiode.value
        if i0 == 0:
            warn = "beamStopDiode is Null --> If we are testing, this is OK, else investigate !!!"
            self.lstExecutiveSummary.append(warn)
            self.warning(warn)
            self.scale = self.experimentSetup.normalizationFactor.value
        else:
            self.scale = self.experimentSetup.normalizationFactor.value / i0


    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.process")

        xsd = XSDataInputWaitFile(expectedFile=XSDataFile(XSDataString(self.rawImage)),
                                           expectedSize=self.rawImageSize,
                                           timeOut=XSDataTime(30))
        self.__edPluginWaitFile.setDataInput(xsd)
        self.__edPluginWaitFile.connectSUCCESS(self.doSuccessWaitFile)
        self.__edPluginWaitFile.connectFAILURE(self.doFailureWaitFile)
        self.__edPluginWaitFile.executeSynchronous()
        if self.isFailure():
            return

        self.xsDataResult.sample = self.sample
        self.xsDataResult.experimentSetup = self.experimentSetup

        q, I, std = self.integrate()
        I = self.normalize(I)
        std = self.normalize(std)
        self.write3ColumnAscii(q, I, std, self.integratedCurve)
        self.xsDataResult.dataQ = EDUtilsArray.arrayToXSData(q)
        self.xsDataResult.dataI = EDUtilsArray.arrayToXSData(I)
        self.xsDataResult.dataStdErr = EDUtilsArray.arrayToXSData(std)

    def postProcess(self, _edObject=None):
        EDPluginControl.postProcess(self)
        self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.postProcess")
        # Create some output data
        if os.path.exists(self.integratedCurve):
            self.xsDataResult.integratedCurve = XSDataFile(XSDataString(self.integratedCurve))

        self.xsDataResult.status = XSDataStatus(executiveSummary=XSDataString(os.linesep.join(self.lstExecutiveSummary)))
        self.setDataOutput(self.xsDataResult)

    def integrate(self):
        img = fabio.open(self.rawImage)
        if "Date" in img.header:
            self.experimentSetup.timeOfFrame = XSDataTime(time.mktime(time.strptime(img.header["Date"], "%a %b %d %H:%M:%S %Y")))
        wavelength = EDUtilsUnit.getSIValue(self.experimentSetup.wavelength)
        current_config = self.integrator.getPyFAI()
        short_config = {}
        for key in self.integrator_config:
            short_config[key] = current_config[key]

        with self.__class__.semaphore:
            if (short_config != self.integrator_config) or \
               (self.integrator.wavelength != wavelength) or\
               (self.maskfile != self.experimentSetup.maskFile.path.value):
                self.screen("Resetting PyFAI integrator")
                self.integrator.setPyFAI(**self.integrator_config)
                self.integrator.wavelength = wavelength
                self.integrator.detector.mask = self.calc_mask()

            q, I, std = self.integrator.integrate1d(img.data, max(img.dim1, img.dim2),
                                       correctSolidAngle=True,
                                       dummy=self.dummy, delta_dummy=self.delta_dummy,
                                       filename=None,
                                       error_model="poisson",
                                       radial_range=None, azimuth_range=None,
                                       polarization_factor=0, dark=None, flat=None,
                                       method=self.METHOD, unit="q_nm^-1", safe=False)
        self.lstExecutiveSummary.append("Azimuthal integration of raw image '%s'-->'%s'." % (self.rawImage, self.integratedCurve))
        return q, I, std

    def normalize(self, data):
        """
        Perform the normalization of some data
        @return: normalized data
        """
        maskedData = numpy.ma.masked_array(data, abs(data - self.dummy) < self.delta_dummy)
        return numpy.ma.filled(maskedData * self.scale, self.dummy)

    def calc_mask(self):
        """
        Merge the natural mask from the detector with the user proided one.
        @return: numpy array with the mask
        """

        mask = fabio.open(self.experimentSetup.maskFile.path.value).data

        detector_mask = pyFAI.detectors.detector_factory(self.detector).calc_mask()
        shape0, shape1 = detector_mask.shape
        if detector_mask.shape == mask.shape:
            mask = numpy.logical_or(mask, detector_mask)
        else:
            # crop the user defined mask
            mask = numpy.logical_or(mask[:shape0, :shape1], detector_mask)
        self.__class__.maskfile = self.experimentSetup.maskFile.path.value
        return mask

    def write3ColumnAscii(self, npaQ, npaI, npaStd=None, outputCurve="output.dat", hdr="#", linesep=os.linesep):
        """
        @param npaQ,npaI,npaStd: 3x 1d numpy array containing Scattering vector, Intensity and deviation
        @param outputCurve: name of the 3-column ascii file to be written
        @param hdr: header mark, usually '#'


Adam Round explicitelly asked for (email from Date: Tue, 04 Oct 2011 15:22:29 +0200) :
Modification from:

# BSA buffer
# Sample c= 0.0 mg/ml (these two lines are required for current DOS pipeline and can be cleaned up once we use EDNA to get to ab-initio models)
#
# Sample environment:
# Detector = Pilatus 1M
# PixelSize_1 = 0.000172
# PixelSize_2 = 6.283185 (I think it could avoid confusion if we give teh actual pixel size as 0.000172 for X and Y and not to give the integrated sizes. Also could there also be a modification for PixelSize_1 as on the diagonal wont it be the hypotenuse (0.000243)? and thus will be on average a bit bigger than 0.000172)
#
# title = BSA buffer
# Frame 7 of 10
# Time per frame (s) = 10
# SampleDistance = 2.43
# WaveLength = 9.31e-11
# Normalization = 0.0004885
# History-1 = saxs_angle +pass -omod n -rsys normal -da 360_deg -odim = 1 /data/id14eh3/inhouse/saxs_pilatus/Adam/EDNAtests/2d/dumdum_008_07.edf/data/id14eh3/inhouse/saxs_pilatus/Adam/EDNAtests/misc/dumdum_008_07.ang
# DiodeCurr = 0.0001592934
# MachCurr = 163.3938
# Mask = /data/id14eh3/archive/CALIBRATION/MASK/Pcon_01Jun_msk.edf
# SaxsDataVersion = 2.40
#
# N 3
# L q*nm  I_BSA buffer  stddev
#
# Sample Information:
# Storage Temperature (degrees C): 4
# Measurement Temperature (degrees C): 10
# Concentration: 0.0
# Code: BSA
s-vector Intensity Error
s-vector Intensity Error
s-vector Intensity Error
s-vector Intensity Error
        """
        hdr = str(hdr)
        headers = []
        if self.sample.comments is not None:
            headers.append(hdr + " " + self.sample.comments.value)
        else:
            headers.append(hdr)
        if self.sample.concentration is not None:
            headers.append(hdr + " Sample c= %s mg/ml" % self.sample.concentration.value)
        else:
            headers.append(hdr + " Sample c= -1  mg/ml")
        headers += [hdr,
                   hdr + " Sample environment:"]
        if self.experimentSetup.detector is not None:
            headers.append(hdr + " Detector = %s" % self.experimentSetup.detector.value)
        if self.experimentSetup.pixelSize_1 is not None:
            headers.append(hdr + " PixelSize_1 = %s" % self.experimentSetup.pixelSize_1.value)
        if self.experimentSetup.pixelSize_2 is not None:
            headers.append(hdr + " PixelSize_2 = %s" % self.experimentSetup.pixelSize_2.value)
        headers.append(hdr)
        if self.sample.comments is not None:
            headers.append(hdr + " title = %s" % self.sample.comments.value)
        if (self.experimentSetup.frameNumber is not None) and\
           (self.experimentSetup.frameMax is not None):
            headers.append(hdr + " Frame %s of %s" % (self.experimentSetup.frameNumber.value, self.experimentSetup.frameMax.value))
        if self.experimentSetup.exposureTime is not None:
            headers.append(hdr + " Time per frame (s) = %s" % self.experimentSetup.exposureTime.value)
        if self.experimentSetup.detectorDistance is not None:
            headers.append(hdr + " SampleDistance = %s" % self.experimentSetup.detectorDistance.value)
        if self.experimentSetup.wavelength is not None:
            headers.append(hdr + " WaveLength = %s" % self.experimentSetup.wavelength.value)
        if self.experimentSetup.normalizationFactor is not None:
            headers.append(hdr + " Normalization = %s" % self.experimentSetup.normalizationFactor.value)
        if self.experimentSetup.beamStopDiode is not None:
            headers.append(hdr + " DiodeCurr = %s" % self.experimentSetup.beamStopDiode.value)
        if self.experimentSetup.machineCurrent is not None:
            headers.append(hdr + " MachCurr = %s" % self.experimentSetup.machineCurrent.value)
        if self.experimentSetup.maskFile is not None:
            headers.append(hdr + " Mask = %s" % self.experimentSetup.maskFile.path.value)
        headers.append(hdr)
        headers.append(hdr + " N 3")
        if self.sample.comments is not None:
            headers.append(hdr + " L q*nm  I_%s  stddev" % self.sample.comments.value)
        else:
            headers.append(hdr + " L q*nm  I_  stddev")
        headers.append(hdr)
        headers.append(hdr + " Sample Information:")
        if self.experimentSetup.storageTemperature is not None:
            headers.append(hdr + " Storage Temperature (degrees C): %s" % self.experimentSetup.storageTemperature.value)
        if self.experimentSetup.exposureTemperature is not None:
            headers.append(hdr + " Measurement Temperature (degrees C): %s" % self.experimentSetup.exposureTemperature.value)

        if self.sample.concentration is not None:
            headers.append(hdr + " Concentration: %s" % self.sample.concentration.value)
        else:
            headers.append(hdr + " Concentration: -1")
        if self.sample.code is not None:
            headers.append(hdr + " Code: %s" % self.sample.code.value)
        else:
            headers.append(hdr + " Code: ")

        with open(outputCurve, "w") as f:
            f.writelines(linesep.join(headers))
            f.write(linesep)
            if npaStd is None:
                data = ["%14.6e %14.6e " % (q, I)
                        for q, I in zip(npaQ, npaI)
                        if abs(I - self.dummy) > self.delta_dummy]
            else:
                data = ["%14.6e %14.6e %14.6e" % (q, I, std)
                        for q, I, std in zip(npaQ, npaI, npaStd)
                        if abs(I - self.dummy) > self.delta_dummy]
            data.append("")
            f.writelines(linesep.join(data))
            f.flush()

    def doSuccessWaitFile(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.doSuccessWaitFile")
        self.retrieveSuccessMessages(_edPlugin, "EDPluginBioSaxsProcessOneFilev1_4.doSuccessWaitFile")

    def doFailureWaitFile(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.doFailureWaitFile")
        self.retrieveFailureMessages(_edPlugin, "EDPluginBioSaxsProcessOneFilev1_4.doFailureWaitFile")
        self.lstExecutiveSummary.append("Timeout in waiting for file '%s'" % (self.rawImage))
        self.setFailure()
Exemplo n.º 16
0
class EDUtilsPath:
    """
    This is a static utility class for handling of paths.
    """

    EDNA_HOME = dirname(dirname(dirname(abspath(__file__))))
    os.environ["EDNA_HOME"] = EDNA_HOME
    _EDNA_SITE = None
    _EDNA_USERTEMPFOLDER = None
    _EDNA_TESTIMAGES = None
    _EDNA_TESTS = None
    _EDNA_TESTDATA = None
    _USER = os.getenv("USER", "UndefindedUser")
    _TMPDIR = os.getenv("TMPDIR", tempfile.gettempdir())

    _EDNA_PROJECTS = {}  # key: project name. Value: path to the project root
    _EDNA_PLUGINCACHE = None

    __semaphore = Semaphore()

    @classmethod
    def appendListOfPaths(cls, _strPath, _listOfPaths):
        """
        Appends a list of paths to a path, for example [ "..", "..", "..", "..", "src" ],
        and returns the absolute path.
        """

        return abspath(os.path.join(_strPath, *_listOfPaths))

    @classmethod
    def mergePath(cls, _strPath1, _strPath2):
        """
        Merges two paths and returns the absolute path.
        Deprecated: please use 'os.path.join' instead
        """
        EDVerbose.log(
            "EDUtilsPath.mergePath is deprecated, please use 'os.path.join' instead"
        )
        strNewPath = os.path.join(_strPath1, _strPath2)
        return abspath(strNewPath)

    @classmethod
    def getAbsolutePath(cls, _strPath):
        """
        Returns the absolute path.
        Deprecated: please use 'os.path.abspath' instead
        """
        EDVerbose.log(
            "EDUtilsPath.getAbsolutePath is deprecated, please use 'os.path.abspath' instead"
        )
        return abspath(_strPath)

    @classmethod
    def existPath(cls, _strPath):
        """
        Checks if a folder exists.
        Deprecated: please use 'os.path.exists' instead
        """
        EDVerbose.log(
            "EDUtilsPath.existPath is deprecated, please use 'os.path.exists' instead"
        )
        return exists(_strPath)

    @classmethod
    def getEdnaHome(cls):
        """
        Returns the EDNA_HOME variable from environment if not already defined
        """
        return cls.EDNA_HOME

    @classmethod
    def setEdnaHome(cls, _strEdnaHome):
        """
        Sets EDNA_HOME home
        """
        cls.EDNA_HOME = _strEdnaHome

    @classmethod
    def getEdnaSite(cls):
        """
        Returns the EDNA_SITE variable from environment
        """
        # EDNA Site
        if (cls._EDNA_SITE == None):
            cls._EDNA_SITE = os.environ.get("EDNA_SITE")
            if (cls._EDNA_SITE is None):
                EDVerbose.warning(
                    "EDUtilsPath.getEdnaSite: EDNA_SITE not set, using EDNA_SITE='default'"
                )
                cls._EDNA_SITE = "default"
            else:
                EDVerbose.DEBUG(
                    "EDUtilsPath.getEdnaSite: EDNA_SITE is set: %s" %
                    cls._EDNA_SITE)
        return cls._EDNA_SITE

    @classmethod
    def setEdnaSite(cls, _strEdnaSite):
        """
        Sets EDNA_SITE
        """
        cls._EDNA_SITE = _strEdnaSite

    EDNA_SITE = classproperty(getEdnaSite, setEdnaSite)

    @classmethod
    def getCwd(cls):
        """
        Returns the current directory.
        Deprecated: please use 'os.getcwd' instead
        """
        EDVerbose.log(
            "EDUtilsPath.getCwd is deprecated, please use 'os.getcwd' instead")
        return os.getcwd()

    @classmethod
    def createFolder(cls, _strFolderName):
        """
        Creates a folder (directory) if it doesn't already exists.
        This used to be deprecated but IS neverthless thread safe (see bug#681)
        """
        EDVerbose.log("EDUtilsPath.createFolder: %s" % _strFolderName)
        with cls.__semaphore:
            if (not exists(_strFolderName)):
                os.makedirs(_strFolderName)

    @classmethod
    def getFileList(cls, _strPath="./", _strMask="*"):
        """
        Returns a list of file and directory names given a path to a directory.
        """
        if _strMask == "*":
            listFile = os.listdir(_strPath)
        else:
            strFind = os.path.join(_strPath, _strMask)
            listPath = glob.glob(strFind)
            listFile = []
            for strPath in listPath:
                strPath = os.path.basename(strPath)
                listFile.append(strPath)
        return listFile

    @classmethod
    def getFolderName(cls, _str):
        """
        Returns the name of a folder (directory) for a given path.
        Deprecated: please use 'os.path.dirname' instead
        """
        return dirname(_str)

    @classmethod
    def getEdnaUserTempFolder(cls):
        """
        Returns the name of a temporary folder that is unique for a given user.
        """
        if cls._EDNA_USERTEMPFOLDER is None:
            if os.environ.has_key("EDNA_USERTEMPFOLDER"):
                cls._EDNA_USERTEMPFOLDER = os.environ["EDNA_USERTEMPFOLDER"]
                if not os.path.exists(cls._EDNA_USERTEMPFOLDER):
                    EDVerbose.warning(
                        "EDNA_USERTEMPFOLDER environment variable is set to %s put the directory does not exist!"
                        % cls._EDNA_USERTEMPFOLDER)
                    cls._EDNA_USERTEMPFOLDER = None
                elif (not os.access(cls._EDNA_USERTEMPFOLDER, os.W_OK)):
                    EDVerbose.warning(
                        "EDNA_USERTEMPFOLDER environment variable is set to %s put the directory cannot be accessed!"
                        % cls._EDNA_USERTEMPFOLDER)
                    cls._EDNA_USERTEMPFOLDER = None
            if cls._EDNA_USERTEMPFOLDER is None:
                strEdnaTempFileDir = tempfile.gettempdir()
                try:
                    # Working on Windows and Linux:
                    strUserName = getpass.getuser()
                except Exception:
                    # Working on MacOS:
                    strUserName = os.getlogin()
                bIsOk = False
                # Check that we have write access to this directory:
                if os.access(strEdnaTempFileDir, os.W_OK) and os.access(
                        strEdnaTempFileDir, os.X_OK):
                    cls._EDNA_USERTEMPFOLDER = os.path.join(
                        strEdnaTempFileDir, "edna-%s" % strUserName)
                    # Check that we have write access to this directory:
                    if not os.path.exists(cls._EDNA_USERTEMPFOLDER):
                        try:
                            os.mkdir(cls._EDNA_USERTEMPFOLDER)
                        except Exception:
                            EDVerbose.WARNING(
                                "Error when trying to create the directory %s"
                                % cls._EDNA_USERTEMPFOLDER)
                    if os.access(cls._EDNA_USERTEMPFOLDER,
                                 os.W_OK) and os.access(
                                     cls._EDNA_USERTEMPFOLDER, os.X_OK):
                        bIsOk = True
                if not bIsOk:
                    # We cannot use the edna-<user name> folder...
                    EDVerbose.WARNING(
                        "EDUtilsFile.getEdnaUserTempFolder: cannot access user temporary directory %s"
                        % cls._EDNA_USERTEMPFOLDER)
                    # Create temporary directory
                    cls._EDNA_USERTEMPFOLDER = tempfile.mkdtemp(prefix="edna-")
                    EDVerbose.WARNING(
                        "Created temporary directory for this session: %s" %
                        cls._EDNA_USERTEMPFOLDER)
                    EDVerbose.WARNING(
                        "If you would like to continue to use this directory for future sessions"
                    )
                    EDVerbose.WARNING(
                        "please set then environment variable EDNA_USERTEMPFOLDER to %s"
                        % cls._EDNA_USERTEMPFOLDER)
        return cls._EDNA_USERTEMPFOLDER

    @classmethod
    def getEdnaPluginCachePath(cls):
        """This private method initializes the path to the plugin cache file"""
        if cls._EDNA_PLUGINCACHE is None:
            if "EDNA_PLUGINCACHE" in os.environ.keys():
                cls._EDNA_PLUGINCACHE = os.environ["EDNA_PLUGINCACHE"]
                strDirName = os.path.dirname(cls._EDNA_PLUGINCACHE)
                if os.path.exists(cls._EDNA_PLUGINCACHE) and (not os.access(
                        cls._EDNA_PLUGINCACHE, os.W_OK)):
                    EDVerbose.warning(
                        "EDNA_PLUGINCACHE environment variable is set to %s but the file is not writeable!"
                        % cls._EDNA_PLUGINCACHE)
                    cls._EDNA_PLUGINCACHE = None
                elif not os.path.exists(strDirName):
                    EDVerbose.warning(
                        "EDNA_PLUGINCACHE environment variable is set to %s put the parent directory does not exist!"
                        % cls._EDNA_PLUGINCACHE)
                    cls._EDNA_PLUGINCACHE = None
                elif not os.access(strDirName, os.W_OK):
                    EDVerbose.warning(
                        "EDNA_PLUGINCACHE environment variable is set to %s put the parent directory cannot be accessed!"
                        % cls._EDNA_PLUGINCACHE)
                    cls._EDNA_PLUGINCACHE = None
            if cls._EDNA_PLUGINCACHE is None:
                strTempFileDir = EDUtilsPath.getEdnaUserTempFolder()
                # We create a hash of the path in order to be able to reference several different EDNA_HOME
                # caches in the same directory
                strCacheFileName = hashlib.sha1(
                    os.getenv('EDNA_HOME')).hexdigest() + ".xml"
                cls._EDNA_PLUGINCACHE = os.path.abspath(
                    os.path.join(strTempFileDir, strCacheFileName))
            EDVerbose.DEBUG("EDFactoryPlugin: Path to plugin cache: %s" %
                            cls._EDNA_PLUGINCACHE)
        return cls._EDNA_PLUGINCACHE

    @classmethod
    def getEdnaTestPath(cls):
        """
        This class method initializes the path to the test path (probably unused)
        """
        if cls._EDNA_TESTS is None:
            if "EDNA_TESTS" in os.environ.keys():
                cls.setEdnaTestDataPath(os.environ["EDNA_TESTS"])
            if cls._EDNA_TESTS is None:
                cls._EDNA_TESTS = os.path.join(
                    cls.EDNA_HOME,
                    "tests",
                )
            EDVerbose.DEBUG(
                "EDFactoryPlugin: Path to Test Data directory : %s" %
                cls._EDNA_TESTS)
        return cls._EDNA_TESTS

    @classmethod
    def setEdnaTestPath(cls, value):
        """
        This class method initializes the path to the test data (probably unused)
        """
        cls._EDNA_TESTS = os.path.abspath(value)
        if not os.path.isdir(cls._EDNA_TESTS):
            EDVerbose.warning(
                "EDNA_TESTS environment variable is set to %s put the directory does not exist!"
                % cls._EDNA_TESTS)
            cls._EDNA_TESTS = None

    EDNA_TESTS = classproperty(getEdnaTestPath, setEdnaTestPath)

    @classmethod
    def getEdnaTestDataPath(cls):
        """
        This class method initializes the path to the test data (probably unused)
        """
        if cls._EDNA_TESTDATA is None:
            if "EDNA_TESTDATA" in os.environ.keys():
                cls.setEdnaTestDataPath(os.environ["EDNA_TESTDATA"])
            if cls._EDNA_TESTDATA is None:
                cls._EDNA_TESTDATA = os.path.join(cls.EDNA_HOME, "tests",
                                                  "data")
            EDVerbose.DEBUG(
                "EDFactoryPlugin: Path to Test Data directory : %s" %
                cls._EDNA_TESTDATA)
        return cls._EDNA_TESTDATA

    @classmethod
    def setEdnaTestDataPath(cls, value):
        """
        This class method initializes the path to the test data (probably unused)
        """
        cls._EDNA_TESTDATA = os.path.abspath(value)
        if not os.path.isdir(cls._EDNA_TESTDATA):
            EDVerbose.warning(
                "EDNA_TESTDATA environment variable is set to %s put the directory does not exist!"
                % cls._EDNA_TESTDATA)
            cls._EDNA_TESTDATA = None

    EDNA_TESTDATA = classproperty(getEdnaTestDataPath, setEdnaTestDataPath)

    @classmethod
    def getEdnaTestDataImagesPath(cls):
        """
        This class method initializes the path to the test images downloaded from internet
        """
        if cls._EDNA_TESTIMAGES is None:
            if "EDNA_TESTIMAGES" in os.environ.keys():
                cls.setEdnaTestDataImagesPath(os.environ["EDNA_TESTIMAGES"])
            if cls._EDNA_TESTIMAGES is None:
                cls._EDNA_TESTIMAGES = os.path.join(cls.EDNA_HOME, "tests",
                                                    "data", "images")
            EDVerbose.DEBUG(
                "EDFactoryPlugin: Path to Temporary Images directory : %s" %
                cls._EDNA_TESTIMAGES)
        return cls._EDNA_TESTIMAGES

    @classmethod
    def setEdnaTestDataImagesPath(cls, value):
        """
        This class method initializes the path to the test images downloaded from internet
        """
        cls._EDNA_TESTIMAGES = os.path.abspath(value)
        if os.path.exists(cls._EDNA_TESTIMAGES) and (not os.access(
                cls._EDNA_TESTIMAGES, os.W_OK)):
            EDVerbose.warning(
                "EDNA_TESTIMAGES environment variable is set to %s but the file is not writeable!"
                % cls._EDNA_TESTIMAGES)
            cls._EDNA_TESTIMAGES = None
        elif not os.path.exists(cls._EDNA_TESTIMAGES):
            EDVerbose.warning(
                "EDNA_TESTIMAGES environment variable is set to %s put the directory does not exist!"
                % cls._EDNA_TESTIMAGES)
            cls._EDNA_TESTIMAGES = None
        elif not os.access(cls._EDNA_TESTIMAGES, os.W_OK):
            EDVerbose.warning(
                "EDNA_TESTIMAGES environment variable is set to %s put the directory cannot be accessed!"
                % cls._EDNA_TESTIMAGES)
            cls._EDNA_TESTIMAGES = None

    EDNA_TESTIMAGES = classproperty(getEdnaTestDataImagesPath,
                                    setEdnaTestDataImagesPath)

    @classmethod
    def getDictOfPaths(cls):
        """
        Return an os.environ like dict enriched with internal path 
        """
        res = {
            "${EDNA_HOME}": cls.EDNA_HOME,
            "${EDNA_SITE}": cls.EDNA_SITE,
            "${EDNA_TESTS}": cls.EDNA_TESTS,
            "${EDNA_TESTDATA}": cls.EDNA_TESTDATA,
            "${EDNA_TESTIMAGES}": cls.EDNA_TESTIMAGES,
            "${USER}": cls._USER,
            "${TMPDIR}": cls._TMPDIR
        }
        return res
Exemplo n.º 17
0
Arquivo: EDObject.py Projeto: kif/edna
class EDObject(object):
    """
    Virtual base class for all EDNA Objects (classes).
    It offers some synchronization and locking capabilities to make the code thread safe.
    """
    __semaphoreId = Semaphore()
    __iId_class = 0
    profiling = {}

    def __init__(self):
        """
        Constructor of the main pure virtual class.
        This constructor implements:
        - the creation of the semaphore
        - definition of timer object (uninitialized as potentially not used)
        """
        object.__init__(self)
        with self.__class__.__semaphoreId:
            self.__class__.__iId_class += 1
            self.__iId = self.__class__.__iId_class
            if self.__class__.__name__ not in self.__class__.profiling:
                self.__class__.profiling[self.__class__.__name__] = []
        self.__semaphore = Semaphore()
        self.__fTimeInit = None
        self.__fTimeEnd = None
        self.__classname = None

    def getId(self):
        return self.__iId

    def getClassName(self):
        """
        Retrieves the name of the class
        @return: the name of the class 
        @rtype: string 
        """
        return self.__class__.__name__

    def synchronizeOn(self):
        """
        This method must be used in together with the method synchronizeOff().
        This method makes the code thread-safe till the method synchronizeOff
        is called.
        """
        self.__semaphore.acquire()

    def synchronizeOff(self):
        """
        This method must be used in together with the method synchronizeOn().
        """
        self.__semaphore.release()

    def getSemaphoreValue(self):
        """
        This method should only be used for debugging purpose...
        @return: the "internal" value of the semaphore
        @rtype: integer
        """
        iValue = self.__semaphore._Semaphore__value
        # EDVerbose.WARNING("DEBUG INFO: The value of semaphore for instance of class %s with hash %s is %i" % (self.getClassName(), hash(self), iValue))
        return iValue

    def locked(self):
        return self.__semaphore

    def setTimeInit(self):
        """
        Initializes the timer for the object
        """
        if self.__fTimeInit is None:
            self.__fTimeInit = time.time()

    def getTimeInit(self):
        """
        Retrieves the time of initialization
        @return: number of seconds since epoch
        @rtype: float  
        """
        return self.__fTimeInit

    def setTimeEnd(self):
        """
        Set the end of calculation time for the given object
        """
        if self.__fTimeEnd is None:
            self.__fTimeEnd = time.time()
            if self.__fTimeInit:
                self.__class__.profiling[self.getClassName()].append(self.__fTimeEnd - self.__fTimeInit)

    def getTimeEnd(self):
        """
        Retrieves the time of end of task
        @return: number of seconds since epoch
        @rtype: float  
        """
        return self.__fTimeEnd

    def getRunTime(self):
        """
        @returns: the RunTime for the given object
        @rtype: float
        """
        fRetrunRunTime = 0.0
        if self.__fTimeInit is not None:
            if self.__fTimeEnd is None:
                fRetrunRunTime = time.time() - self.__fTimeInit
            else:
                fRetrunRunTime = self.__fTimeEnd - self.__fTimeInit
        return fRetrunRunTime

    @classmethod
    def analyze_profiling(cls):
        "Analyse the profiling an return a list of strings to be printed out"
        res = ["Analysis on: %s" % time.asctime(),
               "  Calls | Mean (s) | Std dev  | Total (s) | Plugin name",
               "-" * 80]
        subres = {}
        import numpy
        for name, lst_timimgs in cls.profiling.items():
            npd = numpy.array(lst_timimgs)
            tot = npd.sum()
            line = " %6d | %8.3f | %8.3f | %9.3f | %s " % \
                    (npd.size, tot / npd.size, npd.std(), tot , name)
            subres[tot] = line
        timimgs = list(subres.keys())
        timimgs.sort()
        for key in timimgs:
            res.append(subres[key])
        return res
Exemplo n.º 18
0
class CudaCorrelate(object):
    plans = {}
    data1_gpus = {}
    data2_gpus = {}
    multconj = None
    ctx = None
    #    pycuda.autoinit.context.pop()
    #    ctx.pop()
    sem = Semaphore()
    initsem = Semaphore()

    def __init__(self, shape):
        self.shape = tuple(shape)

    def init(self):
        if self.ctx is None:
            with self.__class__.initsem:
                if self.ctx is None:
                    self.__class__.ctx = pycuda.autoinit.context
        if not self.shape in self.plans:
            with self.__class__.initsem:
                if not self.shape in self.plans:
                    self.ctx.push()
                    if not self.__class__.multconj:
                        self.__class__.multconj = pycuda.elementwise.ElementwiseKernel(
                            "pycuda::complex<double> *a, pycuda::complex<double> *b",
                            "a[i]*=conj(b[i])")
                    if self.shape not in self.__class__.data1_gpus:
                        self.__class__.data1_gpus[self.shape] = gpuarray.empty(
                            self.shape, numpy.complex128)
                    if self.shape not in self.__class__.data2_gpus:
                        self.__class__.data2_gpus[self.shape] = gpuarray.empty(
                            self.shape, numpy.complex128)
                    if self.shape not in self.__class__.plans:
                        self.__class__.plans[self.shape] = cu_fft.Plan(
                            self.shape, numpy.complex128, numpy.complex128)
                    self.ctx.synchronize()
                    self.ctx.pop()

    @classmethod
    def clean(cls):
        with initsem:
            with sem:
                if self.ctx:
                    cls.ctx.push()
                    for plan_name in list(cls.plans.keys()):
                        plan = cls.plans.pop(plan_name)
                        del plan
                    for plan_name in list(cls.data1_gpus.keys()):
                        data = cls.data1_gpus.pop(plan_name)
                        data.gpudata.free()
                        del data
                    for plan_name in cls.data2_gpus.copy():
                        data = cls.data2_gpus.pop(plan_name)
                        data.gpudata.free()
                        del data
                    cls.ctx.pop()
                    cls.ctx = None

    def correlate(self, data1, data2):
        self.init()
        with self.__class__.sem:
            self.ctx.push()
            plan = self.__class__.plans[self.shape]
            data1_gpu = self.__class__.data1_gpus[self.shape]
            data2_gpu = self.__class__.data2_gpus[self.shape]
            data1_gpu.set(data1.astype(numpy.complex128))
            cu_fft.fft(data1_gpu, data1_gpu, plan)
            data2_gpu.set(data2.astype(numpy.complex128))
            cu_fft.fft(data2_gpu, data2_gpu, plan)
            #            data1_gpu *= data2_gpu.conj()
            self.multconj(data1_gpu, data2_gpu)
            cu_fft.ifft(data1_gpu, data1_gpu, plan, True)
            #            self.ctx.synchronize()
            res = data1_gpu.get().real
            self.ctx.pop()
        return res
Exemplo n.º 19
0
class EDParallelExecute(EDLogging):
    """ 
    A class helping to make a multi-threaded application from a plugin name and a list of files. 
    """

    def __init__(self, _strPluginName, _functXMLin, \
                  _functXMLout=None, _functXMLerr=None, \
                  _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
        """
        This is the constructor of the edna plugin launcher.
        
        @param _strPluginName: the name of the ENDA plugin
        @type  _strPluginName: python string
        
        @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. 
        @type  _functXMLin: python function
        
        @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
        @type  _functXMLOut: python function
         
        @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
        @type  _functXMLErr: python function 
        
        @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores  will be auto-detected. 
        @type  _iNbThreads: python integer
        
        @param _fDelay: The delay in seconds between two directories analysis 
        @type  _fDelay: python float
        
        @param _bVerbose:  Do you want the EDNA plugin execution to be verbose ?
        @type  _bVerbose: boolean

        @param _bDebug:  Do you want EDNA plugin execution debug output (OBS! very verbose) ?
        @type  _bDebug: boolean
        """
        EDLogging.__init__(self)
        self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
        EDUtilsParallel.initializeNbThread(self.__iNbThreads)
        ################################################################################
        # #We are not using the one from EDUtilsParallel to leave it to control the number of  execPlugins.
        ################################################################################
        self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
        self.__strPluginName = _strPluginName
        self.__functXMLin = _functXMLin
        self.__functXMLout = _functXMLout
        self.__functXMLerr = _functXMLerr
        self.__strCurrWorkDir = os.getcwd()
        self.__strTempDir = None
        self.__listInputPaths = []
        self.__dictCurrentlyRunning = {}
        if _bVerbose is not None:
            if _bVerbose:
                self.setVerboseDebugOn()
            else:
                self.setVerboseOff()
        if _bDebug is not None:
            if _bDebug:
                self.setVerboseDebugOn()
            else:
                self.setVerboseDebugOff()
        self.__fDelay = _fDelay  #default delay between two directory checks.
        self.__bQuit = False  # To check if we should quit the application
        self.__bIsFirstExecute = True
        signal.signal(signal.SIGTERM, self.handleKill)
        signal.signal(signal.SIGINT, self.handleKill)

    def runEDNA(self,
                _pyListInputPaths=["."],
                _strMode="dirwatch",
                _bNewerOnly=False):
        """
        This method runs the parallel execution on the list of directories.

        @param _pyListInputPaths: the name of the directories to look after.
        @type  _pyListInputPaths: python list of strings
        
        @param _strMode: can be dirwatch, inotify, or OffLine (inotify being not yet implemented)
        @type _strMode: python string

        @param _bNewerOnly: in online mode, process only new files (appearing after the program has started), by default it will process all files then wait for newer files and process them.
        @type _bNewerOnly: boolean
        """
        self.moveToTempDir()
        self.__listInputPaths = _pyListInputPaths
        if _strMode == "dirwatch":
            self.watch_directories(_bNewerOnly)
        elif _strMode == "inotify":
            print "inotify online notify mode not yet implemented"
            raise
        else:  #mode offline
            self.runEdnaFunction(self.__listInputPaths, _bIncludeSubdirs=True)
            self.waitForAllProcessToFinish()

    def moveToTempDir(self):
        """
        Create a temporary directory and put all logs there
        """
        self.__strCurrWorkDir = os.getcwd()
        self.__strTempDir = tempfile.mkdtemp(suffix='.log', prefix='edna-')
        self.screen("The log directory of EDNA will be in " +
                    self.__strTempDir)
        os.chdir(self.__strTempDir)

    def start(self, _strXmlInput):
        """
        Launch EDNA with the given XML stream
        @param _strXmlInput:  XML to be passed to the plugin
        @type  _strXmlInput: python string representing the XML data structure
        """
        jobid = None
        if _strXmlInput not in ["", None]:
            job = EDJob(self.__strPluginName)
            job.setDataInput(_strXmlInput)
            job.connectFAILURE(self.failureJobExecution)
            job.connectSUCCESS(self.successJobExecution)
            job.connectCallBack(self.unregisterJob)
            self.semaphoreNbThreadsAcquire()
            jobid = job.execute()
            self.DEBUG("Running Job id %s" % jobid)
            if jobid is None:
                self.semaphoreNbThreadsRelease()
        return jobid

    def successJobExecution(self, _jobId):
        """
        Method called when the execution of the plugin finishes with success

        @param  _jobId: string of type EDPluginName-number 
        """
        self.DEBUG("EDParallelExcecute.successJobExecution for %s" % _jobId)
        self.semaphoreNbThreadsRelease()
        with self.locked():
            if self.__functXMLout is not None:
                job = EDJob.getJobFromID(_jobId)
                self.__functXMLout(job.getPlugin().getDataInput(),
                                   job.getPlugin().getDataOutput())

    def failureJobExecution(self, _jobId):
        """
        Method called when the execution of the plugin finishes with failure 
        
        @param  _jobId: string of type EDPluginName-number
        """
        self.DEBUG("EDParallelExcecute.failureJobExecution for %s" % _jobId)
        self.semaphoreNbThreadsRelease()
        with self.locked():
            if self.__functXMLerr is not None:
                self.__functXMLerr(
                    EDJob.getJobFromID(_jobId).getPlugin().getDataInput())

    def unregisterJob(self, _jobid):
        """
        remove the filename from the list of files currently under processing

        @param  _jobId: string of type EDPluginName-number          
        """
        with self.locked():
            for oneKey in self.__dictCurrentlyRunning.copy():
                if self.__dictCurrentlyRunning[oneKey] == _jobid:
                    self.__dictCurrentlyRunning.pop(oneKey)

    def runEdnaFunction(self, _listNewFiles, _bIncludeSubdirs=False):
        """
        This method is the launcher for new files found by watch_directories ; it is also called directly in offline mode.
        
        @param  _listNewFiles: list of files newly created in the directory.
        @type   _listNewFiles: python list of strings.
        @param  _bIncludeSubdirs: should we include sub-directories ? yes for offline and no for online.
        @type   _bIncludeSubdirs: boolean
        """
        for oneFile in _listNewFiles:
            if os.path.isdir(oneFile) and _bIncludeSubdirs == True:
                for root, _, onesubdirfiles in os.walk(oneFile):
                    for onesubdirfile in onesubdirfiles:
                        strFilename = os.path.abspath(
                            os.path.join(root, onesubdirfile))
                        if self.__bQuit == True:
                            return
                        self.processOneFile(strFilename)
            elif os.path.isfile(oneFile):
                if self.__bQuit == True:
                    return
                self.processOneFile(oneFile)

    def processOneFile(self, _strFilename):
        """
        Process on file by calling subsequently the XML generator and the start method unless this file
        is already under process (can happend with the watch_directory method).

        @param  _strFilename: filename to process
        @type _strFilename: string
        """
        if _strFilename not in self.__dictCurrentlyRunning:
            with self.locked():
                self.__dictCurrentlyRunning[
                    _strFilename] = self.__strPluginName
            strXmlData = self.__functXMLin(_strFilename)
            if strXmlData in [None, ""]:
                self.log("Not processing % s" % _strFilename)
                with self.locked():
                    self.__dictCurrentlyRunning.pop(_strFilename)
            else:
                self.screen("Processing % s" % _strFilename)
                jobid = self.start(strXmlData)
                with self.locked():
                    if jobid is None:
                        self.__dictCurrentlyRunning.pop(_strFilename)
                    else:
                        self.__dictCurrentlyRunning[_strFilename] = jobid

    def watch_directories(self, _bNewerOnly=False):
        """
        Continuously monitors the paths and their subdirectories
        for changes.  If any files or directories are modified,
        the callable function ( here the method self.runEdnaFunction() ) is called 
        with a list of the modified paths of both
        files and directories.  This function can return a Boolean value
        for rescanning; if it returns True, the directory tree will be
        rescanned without calling the function for any found changes.
        (This is so this function can write changes into the tree and prevent itself
        from being immediately called again.)

        @param _bNewerOnly : Do you want to process only newer files  
        @type _bNewerOnly  : Boolean

        Basic principle: pyDictAllFiles is a dictionary mapping paths to
        modification times.  We repeatedly crawl through the directory
        tree rooted at 'path', doing a stat() on each file and comparing
        the modification time.

        """
        dictAllFiles = {}
        dictRemainingFiles = {}
        listChangedFiles = []

        def internalUpdateDict(unused, dirname, files):
            "Traversal function for directories"
            for strFilename in files:
                path = os.path.join(dirname, strFilename)

                try:
                    tempStat = os.stat(path)
                except os.error:
                    # If a file has been deleted between os.path.walk()
                    # scanning the directory and now, we'll get an
                    # os.error here.  Just ignore it -- we'll report
                    # the deletion on the next pass through the main loop.
                    continue

                mtime = dictRemainingFiles.get(path)
                if mtime is not None:
                    # Record this file as having been seen
                    del dictRemainingFiles[path]
                    # File's mtime has been changed since we last looked at it.
                    if tempStat.st_mtime > mtime:
                        listChangedFiles.append(path)
                else:
                    # No recorded modification time, so it must be
                    # a brand new file.
                    listChangedFiles.append(path)

                # Record current mtime of file.
                dictAllFiles[path] = tempStat.st_mtime

        if _bNewerOnly:
            for path in self.__listInputPaths:
                os.path.walk(path, internalUpdateDict, None)

        # Main loop
        rescan = False
        while not self.__bQuit:
            listChangedFiles = []
            dictRemainingFiles = dictAllFiles.copy()
            dictAllFiles = {}
            for path in self.__listInputPaths:
                os.path.walk(path, internalUpdateDict, None)
            #removed_list = dictRemainingFiles.keys()
            if rescan:
                rescan = False
            elif listChangedFiles:
                rescan = self.runEdnaFunction(listChangedFiles,
                                              _bIncludeSubdirs=False)
            time.sleep(self.__fDelay)
        print "Quitting the online mode."

    def handleKill(self, signum, frame):
        """ 
        This method is launched when the program catches ctrl-c or get killed. It initialize the exit of the program
        """
        self.__bQuit = True
        sys.stderr.write("Exit requested by signal %s with frame %s.\n" %
                         (signum, frame))
        self.waitForAllProcessToFinish()
        os.chdir(self.__strCurrWorkDir)

    def flush(self):
        """
        This method calls the functXMLin a few times with a flush=True argument or without arguments and finishes the work 
        """
        bFinished = False
        while not bFinished:
            xml = None

            try:
                xml = self.__functXMLin(None, flush=True)
            except TypeError:
                try:
                    xml = self.__functXMLin("", flush=True)
                except TypeError:
                    try:
                        xml = self.__functXMLin("")
                    except TypeError:
                        try:
                            xml = self.__functXMLin("")
                        except TypeError:
                            xml = None
        if (xml is None) or (xml == ""):
            bFinished = True
        else:
            self.screen("Flushing data ...")
            self.start(xml)

    def waitForAllProcessToFinish(self):
        """
        as it names says, this method waits for all plug-ins which are currently running to finish before returning.
        """
        self.screen("Waiting for launched jobs to finish .")
        while (self.getNbRunning() > 0):
            time.sleep(1)
            sys.stderr.write(".")
        sys.stderr.write("Done.\n")
        EDJob.stats()

    def cleanUp(self, listMethods=[]):
        """
        Final hook if you need to execute something after all processes finished (like killAllWorkers in SPD) 
        @param listMethods: allows to finish some things in the plugin. 
        @type listMethods: list of strings representing names of methods of the plugin to be called.
        """
        self.waitForAllProcessToFinish()
        for strOneMethod in listMethods:
            try:
                print "calling edPlugin.%s" % strOneMethod
                exec "edPlugin.%s" % strOneMethod
            except Exception:
                print "error in processing %s" % strOneMethod

################################################################################
# Nota: there are 2 levels of controls for the number of thread currently running:
# * One here to limit the number of control plugin running at once
# * One on the Exec plugin level for finer grain optimisation
################################################################################

    def semaphoreNbThreadsAcquire(self):
        """Method to acquire the semaphore that controls the number of plugins running concurrently"""
        #        pass
        self.__semaphoreNbThreads.acquire()

    def semaphoreNbThreadsRelease(self):
        """Method to release the semaphore that controls the number of plugins running concurrently"""
        #        pass
        self.__semaphoreNbThreads.release()

    def getNbRunning(self):
        """
        Class method:
        getter for the number of CPU-active threads running
    
        @return: the number of CPU-active threads runnings
        @rtype: integer
        """
        #return EDUtilsParallel.getNbRunning()
        return self.__iNbThreads - self.__semaphoreNbThreads._Semaphore__value
Exemplo n.º 20
0
class EDJob(EDLogging):
    """
    Create a module called EDJob
    * Most of what was done up to 09-2010 in EDParallelExecute should be done here
    * Each instance will be a job
    * Constructor takes a plugin name
    * Each instance will have taking an "setDataInput" method getting an XMLin (as string)
    * Each instance will gave a "getDataOutput" method with optional join 
    * there could be a "join" method, waiting for the job to finish
    * Each instance will have a "execute" method  and  returning a JobId 
    * Each instance will have a "setCallBack" method  that stores the name of the external callback 
    * provide status of a job
    * keep track of all plugin status
    * leave the time to plugin to initialize
    * static class retrieve job-instance, status, small-log ...
    * prevent multiple run of a single job ?
    * does not manage workload of the computer, should be managed at the ExecPlugin level
    
    Used for the tango binding, EDParallelExecute ...
    
    == class variables ==
    dictPluginStatus[pluginName] = ["uninitialized"|"running"|"executed"| "failed"]
    dictJobs [JobId] = EDJob.Instance
    
    == static methods ==
    getJob(JobId)
    """
    PLUGIN_STATE_UNITIALIZED = "uninitialized"
    PLUGIN_STATE_RUNNING = "running"
    PLUGIN_STATE_SUCCESS = "success"
    PLUGIN_STATE_FAILURE = "failure"

    __edFactoryPlugin = EDFactoryPlugin()
    __dictJobs = {}
    __semaphore = Semaphore()
    __fStartTime = time.time()

    def __init__(self, _strPluginName):
        """
        Constructor of the class
        
        @param strPluginName: name of the plugin 
        @type strPluginName: string
        """
        EDLogging.__init__(self)
        self.__strPluginName = _strPluginName
        self.__edPlugin = None
        self.__edSlotCallBack = EDSlot()
        self.__edSlotSUCCESS = EDSlot()
        self.__edSlotFAILURE = EDSlot()
        self.__pathXSDInput = None
        self.__pathXSDOutput = None
        self.__bXmlInputSet = False
        self.__status = None
        self.__name = None
        self.__runtime = None
        self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(
            self.__strPluginName)
        if self.__edPlugin is None:
            raise RuntimeError("Unable to create plugin %s" %
                               self.__strPluginName)
        self.__jobId = "%s-%08i" % (self.__strPluginName,
                                    self.__edPlugin.getId())
        with self.__class__.__semaphore:
            self.__class__.__dictJobs[self.__jobId] = self
        if (self.__edPlugin is None):
            self.WARNING("Instantiation of plugin %s failed!!!" %
                         _strPluginName)
        else:
            self.__status = EDJob.PLUGIN_STATE_UNITIALIZED

    def setDataInput(self, _oDataInput, _strDataInputKey=None):
        """
        Sets the job (plugin) input data.
        
        @param: _oDataInput: could be either an String XML or an XSData object.
        @param _strDataInputKey: the key of an input data dictionnary
        
        The input data is stored in a dictionary with the key _strDataInputKey.
        If the key is not provided a default key is used.

        If not data input class is defined for the key an exception is raised.
        
        If the key is not the default key, the data object is added to a list which 
        might contain already stored object(s).
        
        If _oDataInput is None the list corresponding to a keyword is deleted.
        """

        if _oDataInput in ["", None]:
            self.__bXmlInputSet = False
            return
        else:
            with self.locked():
                if (self.__edPlugin is not None):
                    self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey)
                    self.__bXmlInputSet = True
                else:
                    self.WARNING(
                        "Setting DataInput for uninstanciated plugin %s." %
                        self.__strPluginName)

    def getDataInput(self, _strDataInputKey=None):
        """
        Returns the Plugin Input Data for a particular key.
        If the key is not provided a default key is used.
        """
        if (self.__edPlugin is not None):
            return self.__edPlugin.getDataInput(_strDataInputKey).marshal()
        elif (self.__pathXSDInput is not None):
            return open(self.__pathXSDInput).read()
        else:
            self.WARNING("Getting DataInput for uninstanciated plugin %s." %
                         self.__strPluginName)

    dataInput = property(getDataInput, setDataInput)

    def getDataOutput(self, _strDataOutputKey=None, _bWait=True):
        """
        Returns the Plugin Output Data
        @param _bWait: shall we wait for the plugin to finish to retrieve output data: Yes by default.
        @type _bWait: boolean
        """
        if _bWait:  # Wait for plugin to finish befor returning data output
            self.synchronize()
        if (self.__edPlugin is not None):
            return self.__edPlugin.getDataOutput(_strDataOutputKey).marshal()
        elif self.__pathXSDOutput is not None:
            return open(self.__pathXSDOutput).read()
        else:
            self.WARNING(
                "Getting DataOutput for uninstanciated plugin or plugin has been garbage collected or output data were not saved. JobID was %s ."
                % self.__jobId)

    dataOutput = property(getDataOutput)

    def execute(self):
        """
        Launch the EDNA plugin
        @return: JobId
        @rtype: string
        """
        if not self.__bXmlInputSet:
            self.WARNING("Not executing job %s as input is empty" %
                         self.__jobId)

        if (self.__edPlugin is not None):
            with self.locked():
                self.__edPlugin.connectSUCCESS(self.successPluginExecution)
                self.__edPlugin.connectFAILURE(self.failurePluginExecution)
                self.__status = EDJob.PLUGIN_STATE_RUNNING
                self.__edPlugin.execute()
                return self.__jobId
        else:
            self.WARNING("Trying to run a plugin that does not exist: %s " %
                         self.__strPluginName)

    def synchronize(self):
        """
        Synchronize the execution of the job with the calling thread.
        """
        with self.locked():
            strStatus = self.__status
        if strStatus == EDJob.PLUGIN_STATE_RUNNING:
            self.__edPlugin.synchronize()
        elif strStatus == EDJob.PLUGIN_STATE_UNITIALIZED:
            self.WARNING("Unable to synchronize %s jobs" % strStatus)
        else:
            self.DEBUG("Unable to synchronize %s jobs" % strStatus)

    @classmethod
    def synchronizeAll(cls):
        """
        Wait for all jobs to finish.
        """
        EDVerbose.DEBUG("EDJob.synchronizeAll class method ")
        listJob = cls.__dictJobs.keys()
        for jobid in listJob:
            job = cls.__dictJobs[jobid]
            job.synchronize()
        if len(cls.__dictJobs) != len(listJob):
            EDVerbose.WARNING(
                "EDJob.synchronizeAll: New jobs have been launched while synchronizing"
            )

    def successPluginExecution(self, _edObject=None):
        """
        Method called when the execution of the plugin succeeds 
        """
        with self.locked():
            self.__status = EDJob.PLUGIN_STATE_SUCCESS
            self.screen("Plugin %s: success after %.3fs" %
                        (self.__jobId, _edObject.getRunTime()))
        try:
            self.__edSlotSUCCESS.call(self.__jobId)
        except Exception:
            self.ERROR("Error in execution of Success call-back for %s" %
                       self.__jobId)
            self.writeErrorTrace()
        try:
            self.__edSlotCallBack.call(self.__jobId)
        except Exception:
            self.ERROR(
                "Error in execution of Common call-back (after success) for %s"
                % self.__jobId)
            self.writeErrorTrace()

    def failurePluginExecution(self, _edObject=None):
        """
        Method called when the execution of the plugin failed 
        """
        with self.locked():
            self.__status = EDJob.PLUGIN_STATE_FAILURE
            self.screen("Plugin %s: failure after %.3fs" %
                        (self.__jobId, _edObject.getRunTime()))
        try:
            self.__edSlotFAILURE.call(self.__jobId)
        except Exception:
            self.ERROR("Error in execution of Failure call-back for %s" %
                       self.__jobId)
            self.writeErrorTrace()
        try:
            self.__edSlotCallBack.call(self.__jobId)
        except Exception:
            self.ERROR(
                "Error in execution of Common call-back (after failure) for %s"
                % self.__jobId)
            self.writeErrorTrace()

    def connectSUCCESS(self, _oMethod):
        """
        @param _oMethod: function or method to be called - back
        """

        with self.locked():
            if (_oMethod != None):
                self.__edSlotSUCCESS.connect(_oMethod)

    def connectFAILURE(self, _oMethod):
        """
        @param _oMethod: function or method to be called - back
        """
        with self.locked():
            if (_oMethod != None):
                self.__edSlotFAILURE.connect(_oMethod)

    def connectCallBack(self, _oMethod):
        """
        @param _oMethod: function or method to be called - back
        """
        with self.locked():
            if (_oMethod != None):
                self.__edSlotCallBack.connect(_oMethod)

    def getJobId(self):
        """
        @return: JobId i.e. EDPluginName-Number
        @rtype: string
        """
        return self.__jobId

    jobId = property(getJobId, "EDJob.jobId: read-only property")
    getJobID = getJobId

    def getPluginName(self):
        """
        @return: Name of the plugin
        @rtype: string
        """
        return self.__strPluginName

    pluginName = property(getPluginName,
                          "EDJob.pluginName: read-only property")

    def getPlugin(self):
        """
        @return: the plugin (instance)
        @rtype: python object
        """
        return self.__edPlugin

    plugin = property(getPlugin, "EDJob.plugin: read-only property")

    def getStatus(self):
        """
        @return: status of the Job
        @rtype: string
        """
        return self.__status

    status = property(getStatus, "EDJob.status: read-only property")

    def getName(self):
        return self.__name

    def setName(self, _strName):
        if self.__name is None:
            self.__name = _strName
        else:
            self.WARNING("EDJob.setName: One cannot rename a Job !!!")

    name = property(getName, setName, "EDJob.name: nickname of the job")

    def getMemSize(self):
        """
        try to guess the size in memory of a job
        @return: expected size in memory 
        """
        if asizeof is not None:
            return asizeof.asizeof(self)

    @classmethod
    def getStatusFromID(cls, jobId):
        """
        Retrieve the job (hence the plugin) status
        
        @param jobId: the Job identification number
        @type jobId: string
        @return: the EDJob status 
        @rtype: string 
        """
        if jobId in cls.__dictJobs:
            strRet = cls.__dictJobs[jobId].getStatus()
        else:
            strRet = "Unable to retrieve such job: %s" % jobId
            EDVerbose.WARNING(strRet)
        return strRet

    getStatusFromId = getStatusFromID

    @classmethod
    def getJobFromID(cls, jobId):
        """
        Retrieve the job (hence the plugin)
        
        @param jobId: the Job identification number
        @type jobId: string
        @return: the "EDJob instance", which contains the plugin (__edPlugin) and the status
        @rtype: a Python object. 
        """
        if jobId in cls.__dictJobs:
            return cls.__dictJobs[jobId]
        else:
            EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId)

    getJobFromId = getJobFromID

    @classmethod
    def getMemoryFootprint(cls):
        if asizeof is not None:
            return asizeof.asizesof(cls.__dictJobs)

    def cleanJob(self, forceGC=True):
        """
        Frees the memory associated with the top level plugin
        @param forceGC: Force garbage collection after clean-up
        @type forceGC: boolean
        """
        self.synchronize()
        with self.locked():
            if self.__edPlugin is not None:
                self.__pathXSDOutput = self.__edPlugin.strPathDataOutput
                self.__pathXSDInput = self.__edPlugin.strPathDataInput
                self.__runtime = self.__edPlugin.getRunTime()
                self.__edPlugin = None
        if forceGC:
            gc.collect()

    @classmethod
    def cleanJobfromId(cls, jobId, forceGC=True):
        """
        Frees the memory associated with the top level plugin
        
        @param jobId: the Job identification number
        @type jobId: string
        @param forceGC: Force garbage collection after clean-up
        @type forceGC: boolean
        """
        if jobId in cls.__dictJobs:
            job = cls.__dictJobs[jobId]
            job.cleanJob(forceGC)
            strRet = "Job %s cleaned" % jobId
        else:
            strRet = "Unable to retrieve such EDJob: %s" % jobId
            EDVerbose.WARNING(strRet)
        return strRet

    cleanJobfromID = cleanJobfromId

    @classmethod
    def getDataOutputFromId(cls, jobId):
        """
        Returns the Plugin Output Data
        @param jobId: job idenfier 
        @type jobId: string
        @return: edJob.DataOutput XML string
        """
        output = None
        job = cls.getJobFromId(jobId)
        if job is not None:
            output = job.getDataOutput()
        return output or ""

    getDataOutputFromID = getDataOutputFromId

    @classmethod
    def getDataInputFromId(cls, jobId):
        """
        Returns the Plugin Input Data
        @param jobId: job idenfier 
        @type jobId: string
        @return: edJob.DataInput XML string
        """
        output = None
        job = cls.getJobFromId(jobId)
        if job is not None:
            output = job.getDataInput()
        return output or ""

    getDataInputFromID = getDataInputFromId

    @classmethod
    def countRunning(cls):
        """
        return the number of jobs still running.
        """
        running = 0
        for jobid in list(cls.__dictJobs.keys()):  # python3 !
            running += (
                cls.__dictJobs[jobid].__status == cls.PLUGIN_STATE_RUNNING)
        return running

    @classmethod
    def stats(cls):
        """
        Retrieve some statistics and print them
        """
        lstStrOut = []
        output = []
        fExecTime = time.time() - cls.__fStartTime
        keys = cls.__dictJobs.keys()
        keys.sort()
        for num, key in enumerate(keys):
            job = cls.__dictJobs[key]
            if job.getPlugin() is None:
                runtime = job.__runtime
            else:
                runtime = job.getPlugin().getRunTime()
            output.append(
                [num, key,
                 job.getStatus(), runtime,
                 job.getMemSize()])
        output.sort()
        iNbJob = max(1, len(keys))
        lstStrOut.append("_" * 110)
        lstStrOut.append(
            "%s\t|\t%s\t\t\t\t|\t%s\t|\t%s\t\t|\t%s" %
            ("nr", "EDPluginName-Id", "status", "runtime", "memory"))
        lstStrOut.append("_" * 110)
        fWall = 0.0
        fSumProd = 0.0
        fSumX = 0.0
        fSumX2 = 0.0
        for oneJob in output:
            fWall += oneJob[3]
            fSumX += oneJob[0]
            fSumX2 += oneJob[0] * oneJob[0]
            fSumProd += oneJob[0] * oneJob[3]
            lstStrOut.append("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" %
                             tuple(oneJob))
        lstStrOut.append("_" * 110)
        lstStrOut.append(
            "Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f"
            % (fWall, fExecTime, fWall / fExecTime))
        lstStrOut.append(
            "Average execution time (Wall/N): %.3fs, Average throughput: %.3fs"
            % (fWall / iNbJob, fExecTime / iNbJob))
        if len(keys) > 1:
            fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 -
                                                            fSumX * fSumX)
            fOrd = (fWall - fSlope * fSumX) / iNbJob
        else:
            fSlope = 0.0
            fOrd = fWall
        lstStrOut.append(
            "Regression of execution time: ExecTime = %.3f + %f * NbJob" %
            (fOrd, fSlope))
        strOutput = os.linesep.join(lstStrOut)
        EDVerbose.screen(strOutput)
        return strOutput
Exemplo n.º 21
0
class EDApplication(object):
    """
    This is the main EDNA application class. This class can be sub-classed for any specific application need.
    An EDNA application is able to launch an entry point plugin. It accepts the following parameter:
    --execute    : name of the plugin to be executed 
    --inputFile  : related plugin data (xml input data file name)
    --outputFile : related plugin result (xml output data file name)
    --conf       : configuration file name
    --basedir    : where the application working directory should go 
    --DEBUG or --debug : turns on debugging   
    -v or --version : Displays the application name and version
    --verbose    : Turns on verbose mode
    --no-log     : Turns off logging
    -h or --help : Prints out an usage message
    """

    CONFIGURATION_PARAM_LABEL = "--conf"
    PLUGIN_PARAM_LABEL = "--execute"
    DATASET_PARAM_LABEL = "--inputFile"
    OUTPUT_PARAM_LABEL = "--outputFile"
    DATASET_BASE_DIRECTORY = "--basedir"
    DEBUG_PARAM_LABEL_1 = "--DEBUG"
    DEBUG_PARAM_LABEL_2 = "--debug"
    VERSION_PARAM_LABEL_1 = "-v"
    VERSION_PARAM_LABEL_2 = "--version"
    VERBOSE_MODE_LABEL = "--verbose"
    NO_LOG_LABEL = "--no-log"
    HELP_LABEL_1 = "-h"
    HELP_LABEL_2 = "--help"

    __edConfiguration = None
    __edFactoryPlugin = None
    __semaphore = Semaphore()


    def __init__(self, _strName="EDApplication", \
                  _strVersion="1.0.1", \
                  _strPluginName=None, \
                  _strConfigurationFileName=None, \
                  _strDataInputFilePath=None, \
                  _edLogFile=None, \
                  _strBaseDir=None, \
                  _strWorkingDir=None, \
                  _strDataOutputFilePath=None):
        self.__strName = _strName
        self.__strVersion = _strVersion
        self.__strPluginName = _strPluginName
        self.__strConfigurationFileName = _strConfigurationFileName
        self.__strDataInputFilePath = _strDataInputFilePath
        self.__strDataOutputFilePath = _strDataOutputFilePath
        self.__edLogFile = _edLogFile
        self.__strBaseDir = _strBaseDir
        self.__strWorkingDir = _strWorkingDir
        self.__strFullApplicationWorkingDirectory = None
        self.__strXMLData = None
        self.__listErrorMessages = []
        self.__listWarningMessages = []
        self.__xsDataOutput = None
        self.__edObtainedOutputDataFile = None
        self.__strDataOutputFilePath = None
        self.__edPlugin = None
        self.__edCommandLine = EDCommandLine(sys.argv)
        self.__strApplicationInstanceName = self.__strName + "_" + time.strftime("%Y%m%d-%H%M%S", time.localtime(time.time()))
        self.__strLogFileName = self.__strApplicationInstanceName + ".log"
        self.__bIsFailure = False
        self.__strCurrentWorkingDirectory = os.getcwd()
        self.__strConfigurationHome = None
        self.__strPathToLogFile = None


    def execute(self):
        """
        This is the main execute method which executes preProcess, process and postProcess.
        """
        self.preProcess()
        self.process()
        self.postProcess()


    def preProcess(self):
        """
        Creates the application working directory (log dir)
        Initializes the configuration
        retrieves the plugin xml data to be passed to the plugin
        """
        EDVerbose.DEBUG("EDApplication.preProcess")
        self.processCommandline()
        if (not self.__bIsFailure):
            # Check that the plugin can be located
            strPluginLocation = EDFactoryPluginStatic.getFactoryPlugin().getModuleLocation(self.__strPluginName)
            if (strPluginLocation is None):
                EDVerbose.error("Plugin  %s cannot be loaded!" % self.__strPluginName)
                self.__bIsFailure = True
            # Check that the input file can be read
            if (self.getDataInputFilePath() is not None) and (not os.path.exists(self.__strDataInputFilePath)):
                EDVerbose.error("Input XML file not found : %s" % self.__strDataInputFilePath)
                self.__bIsFailure = True
            # Check that the output file can be created
            if (self.__strDataOutputFilePath is not None):
                strOutputDirectory = os.path.dirname(self.__strDataOutputFilePath)
                if (strOutputDirectory is None or strOutputDirectory == ""):
                    strOutputDirectory = os.getcwd()
                    self.__strDataOutputFilePath = os.path.join(strOutputDirectory, self.__strDataOutputFilePath)
                if (not os.access(strOutputDirectory, os.W_OK)):
                    EDVerbose.error("Output directory not writable: %s" % strOutputDirectory)
                    self.__bIsFailure = True
                elif (os.path.exists(self.__strDataOutputFilePath)):
                    if (not os.access(self.__strDataOutputFilePath, os.W_OK)):
                        EDVerbose.error("Output file not writable: %s" % self.__strDataOutputFilePath)
                        self.__bIsFailure = True
        if (not self.__bIsFailure):
            EDVerbose.DEBUG("EDApplication.PLUGIN_PARAM_LABEL: " + EDApplication.PLUGIN_PARAM_LABEL)

            # Load the configuration file
            if (os.path.exists(self.__strConfigurationFileName)):
                EDVerbose.screen("Loading Configuration file: %s" % self.__strConfigurationFileName)
                EDConfigurationStatic.addConfigurationFile(self.__strConfigurationFileName, _bReplace=True)
                pyDictionary = {}
                pyDictionary[ "${EDNA_HOME}" ] = EDUtilsPath.getEdnaHome()
                if self.getDataInputFilePath() is not None:
                    self.__strXMLData = EDUtilsFile.readFileAndParseVariables(self.getDataInputFilePath(), pyDictionary)
            else:
                EDVerbose.warning("Cannot find configuration file: %s" % self.__strConfigurationFileName)
            # Create the application working directory    
            if(self.__strWorkingDir is None):
                self.__strWorkingDir = self.__strApplicationInstanceName
            self.createApplicationWorkingDirectory()


    def process(self):
        """
        Calls the Plugin to be executed
        """
        if (not self.__bIsFailure):
            self.__edPlugin = EDFactoryPluginStatic.loadPlugin(self.__strPluginName)
            if(self.__edPlugin is not None):
                self.__edPlugin.setBaseDirectory(self.__strFullApplicationWorkingDirectory)
                self.__edPlugin.setBaseName(self.__strPluginName)
                self.__edPlugin.setDataInput(self.__strXMLData)
                self.__edPlugin.connectSUCCESS(self.doSuccessActionPlugin)
                self.__edPlugin.connectFAILURE(self.doFailureActionPlugin)
                EDVerbose.DEBUG("EDApplication.process: Executing " + self.__strPluginName)
                self.__edPlugin.execute()
                self.__edPlugin.synchronize()
            else:
                EDVerbose.error(EDMessage.ERROR_PLUGIN_NOT_LOADED_02 % ('EDApplication.process', self.__strPluginName))
                self.__bIsFailure = True




    def processCommandline(self):
        """
        This method is intended to be overridden by applications who
        would like to implement their own command line handling.
        
        This default method implements the following workflow:
            - Check for debug, verbose and log file command line options
        
        """
        EDVerbose.DEBUG("EDApplication.execute")
        EDVerbose.log(self.__edCommandLine.getCommandLine())
        self.processCommandLineDebugVerboseLogFile()
        # Determine the base directory
        if(self.__strBaseDir is None):
            self.processCommandLineBaseDirectory()
        # Set the name of the log file
        self.__strPathToLogFile = os.path.abspath(os.path.join(self.__strBaseDir, self.__strLogFileName))
        EDVerbose.setLogFileName(self.__strPathToLogFile)
        self.processCommandLineHelp()
        if (not self.__bIsFailure):
            self.processCommandLineVersion()
        if (not self.__bIsFailure):
            # Name of the plugin to be executed        
            if (self.__strPluginName is None):
                self.processCommandLinePluginName()
            # Path to the input XML file
            if (self.__strDataInputFilePath is None):
                self.processCommandLineInputFilePath()
            # Path to the output XML file
            if(self.__strDataOutputFilePath is None):
                self.processCommandLineOutputFilePath()
            if (self.__bIsFailure):
                self.usage()
        if (not self.__bIsFailure):
            # If strConfigurationFileName is None, this means that it has not been given to the constructor\
            # It has been given by the command line\
            if(self.__strConfigurationFileName is None):
                self.__strConfigurationFileName = self.getCommandLineArgument(EDApplication.CONFIGURATION_PARAM_LABEL)



    def processCommandLineDebugVerboseLogFile(self):
        EDVerbose.DEBUG("EDApplication.processCommandLineDebugVerboseLogFile")
        EDVerbose.setVerboseOff()
        # Check if no log file
        if (self.__edCommandLine.existCommand(EDApplication.NO_LOG_LABEL)):
            EDVerbose.setLogFileOff()
            EDVerbose.DEBUG("Log file output switched off")
        # Check if debug mode
        if (self.__edCommandLine.existCommand(EDApplication.DEBUG_PARAM_LABEL_1) or
            self.__edCommandLine.existCommand(EDApplication.DEBUG_PARAM_LABEL_2)):
            EDVerbose.setVerboseDebugOn()
            EDVerbose.DEBUG("Debug Mode [ON]")
        # Check if verbose
        if (self.__edCommandLine.existCommand(EDApplication.VERBOSE_MODE_LABEL)):
            EDVerbose.setVerboseOn()


    def processCommandLineHelp(self):
        EDVerbose.DEBUG("EDApplication.processCommandLineHelp")
        if (self.__edCommandLine.existCommand(EDApplication.HELP_LABEL_1)
            or self.__edCommandLine.existCommand(EDApplication.HELP_LABEL_2)):
            EDVerbose.setVerboseOn()
            self.usage()
            self.__bIsFailure = True


    def processCommandLineVersion(self):
        EDVerbose.DEBUG("EDApplication.processCommandLineVersion")
        if (self.__edCommandLine.existCommand(EDApplication.VERSION_PARAM_LABEL_1) or
            self.__edCommandLine.existCommand(EDApplication.VERSION_PARAM_LABEL_2)):
            EDVerbose.setVerboseOn()
            EDVerbose.screen("%s version %s" % (self.__strName, self.__strVersion))
            self.__bIsFailure = True



    def processCommandLinePluginName(self):
        """
        """
        EDVerbose.DEBUG("EDApplication.processCommandLinePluginName")
        if (not self.__edCommandLine.existCommand(EDApplication.PLUGIN_PARAM_LABEL)):
            EDVerbose.error("No %s command line argument found!" % EDApplication.PLUGIN_PARAM_LABEL)
            self.__bIsFailure = True
        else:
            self.__strPluginName = self.getCommandLineArgument(EDApplication.PLUGIN_PARAM_LABEL)
            EDVerbose.DEBUG("EDApplication.processCommandLinePluginName : %s = %s" % (EDApplication.PLUGIN_PARAM_LABEL, self.__strPluginName))


    def processCommandLineInputFilePath(self):
        """
        """
        EDVerbose.DEBUG("EDApplication.processCommandLineInputFilePath")
        if (not self.__edCommandLine.existCommand(EDApplication.DATASET_PARAM_LABEL)):
            EDVerbose.error("No %s command line argument found!" % EDApplication.DATASET_PARAM_LABEL)
            self.__bIsFailure = True
        else:
            self.__strDataInputFilePath = self.getCommandLineArgument(EDApplication.DATASET_PARAM_LABEL)
            EDVerbose.DEBUG("EDApplication.initApplication : %s = %s" % (EDApplication.DATASET_PARAM_LABEL, self.__strDataInputFilePath))


    def processCommandLineOutputFilePath(self):
        """
        """
        EDVerbose.DEBUG("EDApplication.processCommandLineOutputFilePath")
        if (not self.__edCommandLine.existCommand(EDApplication.OUTPUT_PARAM_LABEL)):
            EDVerbose.DEBUG("No %s command line argument found" % EDApplication.OUTPUT_PARAM_LABEL)
        else:
            self.__strDataOutputFilePath = self.getCommandLineArgument(EDApplication.OUTPUT_PARAM_LABEL)
            EDVerbose.DEBUG("EDApplication.initApplication : %s = %s" % (EDApplication.OUTPUT_PARAM_LABEL, self.__strDataOutputFilePath))


    def processCommandLineBaseDirectory(self):
        """
        """
        EDVerbose.DEBUG("EDApplication.processCommandLineBaseDirectory")
        self.__strBaseDir = self.getCommandLineArgument(EDApplication.DATASET_BASE_DIRECTORY)
        if(self.__strBaseDir is None):
            self.__strBaseDir = os.getcwd()
            EDVerbose.DEBUG("Base directory set to current working directory = %s" % (self.__strBaseDir))
        else:
            EDVerbose.DEBUG("%s = %s" % (EDApplication.DATASET_BASE_DIRECTORY, self.__strBaseDir))





    def postProcess(self):
        """
        """
        # Restore the current working directory 
        os.chdir(self.__strCurrentWorkingDirectory)


    @classmethod
    def usage(cls):
        """
        Print usage...
        """
        EDVerbose.screen("")
        EDVerbose.screen("Usage: ")
        EDVerbose.screen("")
        EDVerbose.screen("%35s : Name of the plugin to be executed" % (cls.PLUGIN_PARAM_LABEL))
        EDVerbose.screen("")
        EDVerbose.screen("%35s : Path to the XML input file" % (cls.DATASET_PARAM_LABEL))
        EDVerbose.screen("")
        EDVerbose.screen("-----------------------------------------------------------------------------------------------------------")
        EDVerbose.screen("")
        EDVerbose.screen(" Additional options available:")
        EDVerbose.screen("")
        EDVerbose.screen("%35s : Path to the file wich will contain the XML output" % (cls.OUTPUT_PARAM_LABEL))
        EDVerbose.screen("")
        EDVerbose.screen("%35s : Base directory, i.e. working directory for the application" % (cls.DATASET_BASE_DIRECTORY))
        EDVerbose.screen("")
        EDVerbose.screen("%35s : Verbose mode" % (cls.VERBOSE_MODE_LABEL))
        EDVerbose.screen("")
        EDVerbose.screen("%35s : XSConfiguration file" % (cls.CONFIGURATION_PARAM_LABEL))
        EDVerbose.screen("")
        EDVerbose.screen("%35s : Executable version info" % (cls.VERSION_PARAM_LABEL_1 + " or " + cls.VERSION_PARAM_LABEL_2))
        EDVerbose.screen("")
        EDVerbose.screen("%35s : DEBUG log traces" % (cls.DEBUG_PARAM_LABEL_1 + " or " + cls.DEBUG_PARAM_LABEL_2))
        EDVerbose.screen("")
        EDVerbose.screen("%35s : No log file" % (cls.NO_LOG_LABEL))
        EDVerbose.screen("")
        EDVerbose.screen("%35s : This help message" % (cls.HELP_LABEL_1 + " or " + cls.HELP_LABEL_2))
        EDVerbose.screen("")


    @classmethod
    def getFactoryPlugin(cls):
        EDVerbose.WARNING("the use of EDclsetFactoryPlugin is deprecated. Please use EDFactoryPluginStatic.getFactoryPlugin instead.")
        return EDFactoryPluginStatic.getFactoryPlugin


    @classmethod
    def loadPlugin(cls, _strPluginName):
        EDVerbose.WARNING("The use of EDApplication.loadPlugin is deprecated. Please use EDFactoryPluginStatic.getFactoryPlugin instead.")
        return EDFactoryPluginStatic.loadPlugin(_strPluginName)


    @classmethod
    def loadModule(cls, _strModuleName):
        EDVerbose.WARNING("The use of EDApplication.loadModule is deprecated. Please use EDFactoryPluginStatic.getFactoryPlugin instead.")
        EDFactoryPluginStatic.loadModule(_strModuleName)


    def getDataInputFilePath(self):
        return self.__strDataInputFilePath


    def getBaseDir(self):
        """
        Getter for base directory
        @return: path of the base directory
        @rtype: string
        """
        return self.__strBaseDir


    def createApplicationWorkingDirectory(self):
        """
        Created the working directory of the application (<date>-<application name>)
        First tries to retrieve the base dir from --basedir option or related parameter from constructor
        Otherwise tries to retrieve it from EDNA_BASE_DIRECTORY environment variable
        Otherwise put the base dir as the current directory
        """
        EDVerbose.DEBUG("EDApplication.createApplicationWorkingDirectory")
        strBaseDirectory = self.getBaseDir()
        strDateTime = time.strftime("%Y%m%d-%H%M%S", time.localtime(time.time()))
        self.__strFullApplicationWorkingDirectory = os.path.join(strBaseDirectory, self.__strWorkingDir)
        # Check that a folder / file with the same name already exists
        if(os.path.exists(self.__strFullApplicationWorkingDirectory) or \
            os.path.exists(self.__strFullApplicationWorkingDirectory)):
            # It does exist so we have to modify the name of the working directory
            iIndex = 1
            bContinueFlag = True
            while (bContinueFlag):
                self.__strFullApplicationWorkingDirectory = os.path.join(strBaseDirectory,
                                                                                        "%s_%d" % \
                                                                                        (strDateTime, \
                                                                                          iIndex))
                if(os.path.isdir(self.__strFullApplicationWorkingDirectory) or \
                    os.path.exists(self.__strFullApplicationWorkingDirectory)):
                    iIndex += 1
                else:
                    bContinueFlag = False
        # Make the directory
        os.mkdir(self.__strFullApplicationWorkingDirectory)
        # Change it to be the current working directory
        os.chdir(self.__strFullApplicationWorkingDirectory)


    def getFullApplicationWorkingDirectory(self):
        return self.__strFullApplicationWorkingDirectory

    def getCurrentWorkingDirectory(self):
        return self.__strCurrentWorkingDirectory


    def doSuccessActionPlugin(self, _edPlugin):
        """
        """
        EDVerbose.DEBUG("EDApplication.doSuccessActionPlugin")
        # Print the potential Warnings and Errors
        self.__listWarningMessages = _edPlugin.getListOfWarningMessages()
        EDVerbose.DEBUG("EDApplication.doSuccessActionPlugin: Plugin %s Successful with : %i Warnings " % (_edPlugin.getPluginName(), len(self.__listWarningMessages)))
        for warningMessage in self.__listWarningMessages:
            EDVerbose.screen(warningMessage)
        self.__listErrorMessages = _edPlugin.getListOfErrorMessages()
        EDVerbose.DEBUG("EDApplication.doSuccessActionPlugin: Plugin %s Successful with : %i Errors" % (_edPlugin.getPluginName(), len(self.__listErrorMessages)))
        for errorMessage in self.__listErrorMessages:
            EDVerbose.error(errorMessage)
        if (_edPlugin.hasDataOutput()):
            xsDataOutput = _edPlugin.getDataOutput()
            if (xsDataOutput is not None and self.__strDataOutputFilePath is not None):
                xsDataOutput.exportToFile(self.__strDataOutputFilePath)
            if (xsDataOutput is not None and self.__edObtainedOutputDataFile is not None):
                xsDataOutput.exportToFile(self.__edObtainedOutputDataFile)


    def doFailureActionPlugin(self, _edPlugin):
        EDVerbose.DEBUG("EDApplication.doFailureActionPlugin")

        # Print the potential Warnings and Errors
        EDVerbose.DEBUG("EDApplication.doFailureActionPlugin: Plugin %s failed" % _edPlugin.getClassName())
        self.__listWarningMessages = _edPlugin.getListOfWarningMessages()
        for warningMessage in self.__listWarningMessages:
            EDVerbose.screen(warningMessage)

        self.__listErrorMessages = _edPlugin.getListOfErrorMessages()
        for errorMessage in self.__listErrorMessages:
            EDVerbose.screen(errorMessage)
        if (_edPlugin.hasDataOutput()):
            xsDataOutput = _edPlugin.getDataOutput()
            if (xsDataOutput is not None and self.__strDataOutputFilePath is not None):
                xsDataOutput.exportToFile(self.__strDataOutputFilePath)
            if (xsDataOutput is not None and self.__edObtainedOutputDataFile is not None):
                xsDataOutput.exportToFile(self.__edObtainedOutputDataFile)


    def getPlugin(self):
        return self.__edPlugin


    def getPluginOutputData(self):
        return self.__xsDataOutput


    def getWarningMessages(self):
        return self.__listWarningMessages


    def getErrorMessages(self):
        return self.__listErrorMessages


    def getEdCommandLine(self):
        return self.__edCommandLine


    def getCommandLine(self):
        return self.__edCommandLine.getCommandLine()


    def getCommandLineArguments(self):
        with self.__class__.__semaphore:
            edCommandLine = self.__edCommandLine.getCommandLine()
        return edCommandLine

    def getCommandLineArgument(self, _strKey):
        with self.__class__.__semaphore:
            strCommandLineArgument = self.__edCommandLine.getArgument(_strKey)
        return strCommandLineArgument

    @classmethod
    def synchronizeOn(cls):
        """
        Lock the whole class
        """
        cls.__semaphore.acquire()


    @classmethod
    def synchronizeOff(cls):
        """
        Unlock the whole class
        """
        cls.__semaphore.release()


    def getApplicationName(self):
        return self.__strName + "-" + self.__strVersion


    def getWorkingDir(self):
        """
        Getter for working dir
        @rtype: string
        @return working dir 
        """
        return self.__strWorkingDir


    def setWorkingDir(self, _strDir):
        """
        Setter for working dir
        @type _strDir: string
        @param _strDir: working dir 
        """
        self.__strWorkingDir = _strDir


    def isFailure(self):
        return self.__bIsFailure


    def setFailure(self, _bFailure):
        self.__bIsFailure = _bFailure


    def getPluginName(self):
        return self.__strPluginName
Exemplo n.º 22
0
class EDPluginBioSaxsSmartMergev1_7(EDPluginControl):
    """
    This plugin takes a set of input data files (1D SAXS) measure
    their differences (versus previous and versus first) and merge those which are equivalent
    v1.1 adds information from AutoSub
    
    Controlled plugins:
     - Execplugins/WaitMultifile
    
     - EdnaSaxs/Atsas/DatAver
     - EdnaSaxs/Atsas/AutoSub
     - EdnaSaxs/SaxsAnalysis
    """
    # dictAve = {} #key=?; value=path to average file
    lastBuffer = None
    lastSample = None
    dictFrames = {
    }  #dictionary key = filename of average, value = list of files used for average
    __strControlledPluginDataver = "EDPluginExecDataverv2_0"
    #__strControlledPluginDatcmp = "EDPluginExecDatcmpv2_0"
    __strControlledPluginWaitFile = "EDPluginWaitMultiFile"
    __strControlledPluginAutoSub = "EDPluginAutoSubv1_1"
    __strControlledPluginSaxsAnalysis = "EDPluginControlSaxsAnalysisv1_0"
    __strControlledPluginSaxsISPyB = "EDPluginBioSaxsISPyBv1_0"
    __configured = False
    CONF_MINIMUM_CURVE_FILE_SIZE = "MinCurveFileSize"
    minimumCurveFileSize = 10000
    semaphore = Semaphore()
    cpRsync = "EDPluginExecRsync"

    def __init__(self):
        """
        """
        EDPluginControl.__init__(self)

        #self.__edPluginExecDatcmp = None
        self.__edPluginExecDataver = None
        self.__edPluginExecWaitFile = None
        self.__edPluginExecAutoSub = None
        self.__edPluginSaxsAnalysis = None
        self.__edPluginSaxsISPyB = None
        self.setXSDataInputClass(XSDataInputBioSaxsSmartMergev1_0)
        #self.__edPluginExecDatCmp = None
        self.lstInput = []
        self.curves = []
        self.forgetLastSample = False
        self.lstMerged = []
        self.lstDiscarded = []
        self.lstXsdInput = []
        self.absoluteFidelity = 0.01  #None
        self.relativeFidelity = 0.01  #None
        self.dictSimilarities = {}  # key: 2-tuple of images, similarities
        self.lstStrInput = []
        self.autoRg = None
        self.gnom = None
        self.volume = None
        self.rti = None
        self.strRadiationDamage = None
        self.strMergedFile = None
        self.lstSub = []
        self.strSubFile = None
        self.fConcentration = None
        self.xsDataResult = XSDataResultBioSaxsSmartMergev1_0()
        self.xsBestBuffer = None
        self.bestBufferType = ""
        self.bufferFrames = []
        self.xsScatterPlot = None
        self.xsGuinierPlot = None
        self.xsKratkyPlot = None
        self.xsDensityPlot = None
        self.xsdSubtractedCurve = None
        self.outdir = None  #directory on rnice for analysis results to go to

    def configure(self):
        """
        Configures the plugin from the configuration file with the following parameters:
         - curve_file_size: minimum size of the file.
        """
        EDPluginControl.configure(self)
        if not self.__configured:
            with self.semaphore:
                if not self.__configured:
                    self.DEBUG("EDPluginBioSaxsSmartMergev1_7.configure")
                    min_size = self.config.get(
                        self.CONF_MINIMUM_CURVE_FILE_SIZE)
                    if min_size is None:
                        strMessage = 'EDPluginBioSaxsSmartMergev1_7.configure: %s Configuration parameter missing: \
            %s, defaulting to "%s"' % (self.getBaseName(),
                                       self.CONF_MINIMUM_CURVE_FILE_SIZE,
                                       self.minimumCurveFileSize)
                        self.WARNING(strMessage)
                        self.addErrorWarningMessagesToExecutiveSummary(
                            strMessage)
                    else:
                        self.__class__.minimumCurveFileSize = float(min_size)
                    self.__class__.__configured = True

    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.checkParameters")
        self.checkMandatoryParameters(self.dataInput, "Data Input is None")
        self.checkMandatoryParameters(self.dataInput.inputCurves,
                                      "Input curve list is empty")
        self.checkMandatoryParameters(self.dataInput.mergedCurve,
                                      "Output curve filename  is empty")

    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.preProcess")
        # Load the execution plugin
        if self.dataInput.absoluteFidelity is not None:
            self.absoluteFidelity = self.dataInput.absoluteFidelity.value
            if self.absoluteFidelity == 0.0:
                self.absoluteFidelity = None
        if self.dataInput.relativeFidelity is not None:
            self.relativeFidelity = self.dataInput.relativeFidelity.value
            if self.relativeFidelity == 0.0:
                self.relativeFidelity = None
        self.lstInput = self.dataInput.inputCurves
        self.lstStrInput = [i.path.value for i in self.lstInput]
        self.__edPluginExecWaitFile = self.loadPlugin(
            self.__strControlledPluginWaitFile)

        if self.dataInput.mergedCurve is not None:
            self.strMergedFile = self.dataInput.mergedCurve.path.value

        if self.dataInput.subtractedCurve is not None:
            self.strSubFile = self.dataInput.subtractedCurve.path.value
            dirname = os.path.dirname(self.strSubFile)
            if not os.path.isdir(dirname):
                os.mkdir(dirname)

    def getFramesByFilename(self, filename):
        if filename is not None:
            if filename in self.__class__.dictFrames.keys():
                return self.__class__.dictFrames[filename]
        return {"averaged": [], "discarded": []}

    def getAveragedFrameByFilename(self, filename):
        frames = self.getFramesByFilename(filename)
        if frames is not None:
            return frames["averaged"]
        return []

    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.process")

        xsdwf = XSDataInputWaitMultiFile(
            timeOut=XSDataTime(30),
            expectedSize=XSDataInteger(self.minimumCurveFileSize),
            expectedFile=[XSDataFile(i.path) for i in self.lstInput])
        self.__edPluginExecWaitFile.setDataInput(xsdwf)
        self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
        self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
        self.__edPluginExecWaitFile.executeSynchronous()
        if self.isFailure():
            return
        if len(self.lstInput) == 1:
            inp = self.lstInput[0].path.value
            dst = self.dataInput.mergedCurve.path.value
            if not os.path.isdir(os.path.dirname(dst)):
                self.error("Output directory for %s does not exist" % dst)
                os.makedirs(os.path.dirname(dst))
            if not os.path.exists(inp):
                self.warning("Input %s does not (yet?) exist" % inp)
                time.sleep(1.0)
            shutil.copyfile(inp, dst)
            self.addExecutiveSummaryLine(
                "Got only one frame ... nothing to merge !!!")
        else:
            self.lstMerged = []

            if (self.absoluteFidelity is not None) or (self.relativeFidelity
                                                       is not None):
                if self.absoluteFidelity is not None:
                    for idx, oneFile in enumerate(self.lstInput[1:]):
                        self.DEBUG("Calculating similarity of 0 and %s" % idx)
                        self.datcmp(self.lstInput[0], oneFile)
#                         edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
#                         xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], oneFile])
#                         edPluginExecAbsoluteFidelity.setDataInput(xsd)
#                         edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
#                         edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
#                         edPluginExecAbsoluteFidelity.execute()
                if (self.relativeFidelity is not None):
                    if (self.absoluteFidelity is None):
                        self.DEBUG("Calculating similarity of 0 and 1")
                        self.datcmp(self.lstInput[0], self.lstInput[1])
#                         edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
#                         xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], self.lstInput[1] ])
#                         edPluginExecAbsoluteFidelity.setDataInput(xsd)
#                         edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
#                         edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
#                         edPluginExecAbsoluteFidelity.execute()
                    if (len(self.lstInput) > 2):
                        for idx, oneFile in enumerate(self.lstInput[2:]):
                            self.DEBUG("Calculating similarity of %s and %s" %
                                       (idx, idx + 1))
                            self.datcmp(self.lstInput[idx + 1], oneFile)
#                             edPluginExecRelativeFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
#                             xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[idx + 1], oneFile])
#                             edPluginExecRelativeFidelity.setDataInput(xsd)
#                             edPluginExecRelativeFidelity.connectFAILURE(self.doFailureExecDatcmp)
#                             edPluginExecRelativeFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
#                             edPluginExecRelativeFidelity.execute()
            self.synchronizePlugins()

            for idx, oneFile in enumerate(self.lstInput):
                if idx == 0:
                    self.lstMerged.append(oneFile)
                elif (self.absoluteFidelity
                      is not None) and (self.relativeFidelity is not None):
                    if (idx - 1, idx) not in self.dictSimilarities:
                        self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
                                   "\n".join([
                                       "%s: %s" %
                                       (key, self.dictSimilarities[key])
                                       for key in self.dictSimilarities
                                   ]))
                        self.resynchronize()

                    if (0, idx) not in self.dictSimilarities:
                        self.ERROR("dict missing %i,%i: \n" % (0, idx) +
                                   "\n".join([
                                       "%s: %s" %
                                       (key, self.dictSimilarities[key])
                                       for key in self.dictSimilarities
                                   ]))
                        self.resynchronize()

                    if (self.dictSimilarities[(0, idx)] >=
                            self.absoluteFidelity) and (self.dictSimilarities[
                                (idx - 1, idx)] >= self.relativeFidelity):
                        self.lstMerged.append(oneFile)

                elif (self.absoluteFidelity is not None):
                    if (0, idx) not in self.dictSimilarities:
                        self.ERROR("dict missing %i,%i: \n" % (0, idx) +
                                   "\n".join([
                                       "%s: %s" %
                                       (key, self.dictSimilarities[key])
                                       for key in self.dictSimilarities
                                   ]))
                        self.resynchronize()

                    if (self.dictSimilarities[(0, idx)] >=
                            self.absoluteFidelity):
                        self.lstMerged.append(oneFile)

                elif (self.relativeFidelity is not None):
                    if (idx - 1, idx) not in self.dictSimilarities:
                        self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
                                   "\n".join([
                                       "%s: %s" %
                                       (key, self.dictSimilarities[key])
                                       for key in self.dictSimilarities
                                   ]))
                        self.resynchronize()

                    if (self.dictSimilarities[(idx - 1, idx)] >=
                            self.relativeFidelity):
                        self.lstMerged.append(oneFile)

                else:
                    self.lstMerged.append(oneFile)
            self.lstMerged.sort(cmp)
            if len(self.lstMerged) != len(self.lstInput):
                self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(
                    self.lstMerged)
                self.WARNING(self.strRadiationDamage)
                self.addExecutiveSummaryLine("WARNING: " +
                                             self.strRadiationDamage)
            self.addExecutiveSummaryLine("Merging files: " + " ".join(
                [os.path.basename(i.path.value) for i in self.lstMerged]))
            if len(self.lstMerged) == 1:
                self.rewriteHeader(self.lstMerged[0].path.value,
                                   self.strMergedFile)
            else:
                self.__edPluginExecDataver = self.loadPlugin(
                    self.__strControlledPluginDataver)
                xsd = XSDataInputDataver(inputCurve=self.lstMerged)
                # outputCurve=self.dataInput.mergedCurve,
                self.__edPluginExecDataver.setDataInput(xsd)
                self.__edPluginExecDataver.connectSUCCESS(
                    self.doSuccessExecDataver)
                self.__edPluginExecDataver.connectFAILURE(
                    self.doFailureExecDataver)
                self.__edPluginExecDataver.executeSynchronous()

        if (self.fConcentration == 0) and (self.strSubFile is not None):
            if (self.__class__.lastBuffer
                    is not None) and (self.__class__.lastSample is not None):
                self.__edPluginExecAutoSub = self.loadPlugin(
                    self.__strControlledPluginAutoSub)
                base = "_".join(
                    os.path.basename(
                        self.__class__.lastSample.path.value).split("_")[:-1])
                suff = os.path.basename(self.strSubFile).split("_")[-1]
                sub = os.path.join(os.path.dirname(self.strSubFile),
                                   base + "_" + suff)
                self.xsdSubtractedCurve = XSDataFile(XSDataString(sub))
                #self.curves.append(xsdSubtractedCurve)
                self.__edPluginExecAutoSub.dataInput = XSDataInputAutoSub(
                    sampleCurve=self.__class__.lastSample,
                    buffers=[
                        self.__class__.lastBuffer, self.dataInput.mergedCurve
                    ],
                    subtractedCurve=self.xsdSubtractedCurve)
                self.__edPluginExecAutoSub.connectSUCCESS(
                    self.doSuccessExecAutoSub)
                self.__edPluginExecAutoSub.connectFAILURE(
                    self.doFailureExecAutoSub)
                self.__edPluginExecAutoSub.executeSynchronous()

                if self.isFailure():
                    return

                self.__edPluginSaxsAnalysis = self.loadPlugin(
                    self.__strControlledPluginSaxsAnalysis)
                self.__edPluginSaxsAnalysis.dataInput = XSDataInputSaxsAnalysis(
                    scatterCurve=self.xsdSubtractedCurve,
                    autoRg=self.autoRg,
                    graphFormat=XSDataString("png"))
                self.__edPluginSaxsAnalysis.connectSUCCESS(
                    self.doSuccessSaxsAnalysis)
                self.__edPluginSaxsAnalysis.connectFAILURE(
                    self.doFailureSaxsAnalysis)
                self.__edPluginSaxsAnalysis.executeSynchronous()

            self.__class__.lastBuffer = self.dataInput.mergedCurve
            #self.__class__.lastSample = None #Information neededfor transfer to ISPyB
            self.forgetLastSample = True
        else:
            self.__class__.lastSample = self.dataInput.mergedCurve

        if self.dataInput.sample and self.dataInput.sample.login and self.dataInput.sample.passwd and self.dataInput.sample.measurementID:
            self.addExecutiveSummaryLine("Registering to ISPyB")
            self.lstDiscarded = list(set(self.lstInput) - set(self.lstMerged))
            self.__class__.dictFrames[self.dataInput.mergedCurve] = {
                'averaged': self.lstMerged,
                'discarded': self.lstDiscarded
            }
            self.__edPluginSaxsISPyB = self.loadPlugin(
                self.__strControlledPluginSaxsISPyB)
            if len(self.lstInput) > 1:
                frameAverage = XSDataInteger(len(self.lstInput))
                frameMerged = XSDataInteger(len(self.lstMerged))
            else:
                frameMerged = frameAverage = XSDataInteger(1)
            self.curves = [XSDataFile(i.path) for i in self.lstInput]
            self.discardedCurves = [
                XSDataFile(i.path) for i in self.lstDiscarded
            ]
            self.mergedCurves = [XSDataFile(i.path) for i in self.lstMerged]

            averageFilePath = None
            if self.strMergedFile is not None:
                averageFilePath = XSDataFile(XSDataString(self.strMergedFile))

            self.sampleFrames = self.getAveragedFrameByFilename(
                self.__class__.lastSample)
            lastSample = None
            if self.__class__.lastSample is not None:
                lastSample = self.__class__.lastSample

            subtractedCurve = None
            if self.xsdSubtractedCurve is not None:
                subtractedCurve = self.xsdSubtractedCurve
            else:
                subtractedCurve = None

            xsdin = XSDataInputBioSaxsISPyBv1_0(
                sample=self.dataInput.sample,
                autoRg=self.autoRg,
                gnom=self.gnom,
                volume=self.volume,
                frameAverage=frameAverage,
                frameMerged=frameMerged,
                curves=self.curves,
                discardedFrames=self.discardedCurves,
                averagedFrames=self.mergedCurves,
                averageFilePath=averageFilePath,
                bufferFrames=self.bufferFrames,
                sampleFrames=self.sampleFrames,
                bestBuffer=self.xsBestBuffer,
                averageSample=lastSample,
                scatterPlot=self.xsScatterPlot,
                guinierPlot=self.xsGuinierPlot,
                kratkyPlot=self.xsKratkyPlot,
                densityPlot=self.xsDensityPlot,
                subtractedFilePath=subtractedCurve
                #                                                     destination=self.dataInput.sample.ispybDestination #duplicate, already in sample
            )
            self.__edPluginSaxsISPyB.dataInput = xsdin
            self.__edPluginSaxsISPyB.connectSUCCESS(self.doSuccessISPyB)
            self.__edPluginSaxsISPyB.connectFAILURE(self.doFailureISPyB)
            self.__edPluginSaxsISPyB.execute()

    # transfer analysis data to correct location on nice
        if self.gnom is not None:
            self.outdir = os.path.join(
                os.path.dirname(os.path.dirname(self.lstStrInput[0])),
                "ednaAnalysis")
            basename = os.path.basename(
                os.path.splitext(self.gnom.gnomFile.path.value)[0])
            self.outdir = os.path.join(self.outdir, basename)
            if not os.path.isdir(self.outdir):
                os.makedirs(self.outdir)
            #self.outFile = os.path.join(outdir, "NoResults.html")
            workingdir = os.path.dirname(self.gnom.gnomFile.path.value)

            self.pluginRsync = self.loadPlugin(self.cpRsync)
            self.pluginRsync.dataInput = XSDataInputRsync(
                source=XSDataFile(XSDataString(workingdir)),
                destination=XSDataFile(XSDataString(self.outdir)),
                options=XSDataString("-avx"))

            self.pluginRsync.connectSUCCESS(self.doSuccessExecRsync)
            self.pluginRsync.connectFAILURE(self.doFailureExecRsync)
            self.pluginRsync.executeSynchronous()

        if self.forgetLastSample:
            #Also redefine dictionary to contain the buffer just processed?
            self.__class__.lastSample = None

    def postProcess(self, _edObject=None):
        EDPluginControl.postProcess(self)
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.postProcess")
        # Create some output data
        self.xsDataResult.mergedCurve = self.dataInput.mergedCurve
        if self.strSubFile is not None and os.path.isfile(self.strSubFile):
            self.xsDataResult.subtractedCurve = XSDataFile(
                XSDataString(self.strSubFile))
        self.xsDataResult.autoRg = self.autoRg
        self.xsDataResult.gnom = self.gnom
        self.xsDataResult.volume = self.volume
        self.xsDataResult.rti = self.rti

    def finallyProcess(self, _edObject=None):
        EDPluginControl.finallyProcess(self)

        self.xsDataResult.status = XSDataStatus(
            message=self.getXSDataMessage(),
            executiveSummary=XSDataString(
                os.linesep.join(self.getListExecutiveSummaryLines())))
        self.dataOutput = self.xsDataResult

    def resynchronize(self):
        """
        Sometimes plugins are not started or not yet finished... 
        """
        time.sleep(1)  # this is just a way to give the focus to other threads
        self.synchronizePlugins()
        self.ERROR("I slept a second, waiting for sub-plugins to finish")

    def rewriteHeader(self,
                      infile=None,
                      output=None,
                      hdr="#",
                      linesep=os.linesep):
        """
        Write the output file with merged data with the good header.

# BSA sample
#Sample c= 0.0 mg/ml
#
# Merged from: file1
# Merged from: file2
# Merged from: file4
#
# Sample Information:
# Concentration: 0.0
# Code: BSA

        @param infile: path of the original data file where original data are taken from
        @param outfile: path of the destination file (by default, from XSD)
        @param hdr: header marker, often "#"
        @param linesep: line separator, usually "\n" or "\r\n" depending on the Operating System 
        @return: None
        """
        Code = Concentration = None
        frameMax = exposureTime = measurementTemperature = storageTemperature = None
        originalFile = self.lstMerged[0].path.value
        headers = [
            line.strip() for line in open(originalFile) if line.startswith("#")
        ]
        Comments = headers[0][1:].strip()
        for line in headers:
            if "title =" in line:
                Comments = line.split("=", 1)[1].strip()
            elif "Comments =" in line:
                Comments = line.split("=", 1)[1].strip()
            elif "Concentration:" in line:
                Concentration = line.split(":", 1)[1].strip()
            elif "Concentration =" in line:
                Concentration = line.split("=", 1)[1].strip()
            elif "Code =" in line:
                Code = line.split("=", 1)[1].strip()
            elif "Code:" in line:
                Code = line.split(":", 1)[1].strip()
            elif "Storage Temperature" in line:
                storageTemperature = line.split(":", 1)[1].strip()
            elif "Measurement Temperature" in line:
                measurementTemperature = line.split(":", 1)[1].strip()
            elif "Time per frame" in line:
                exposureTime = line.split("=", 1)[1].strip()
            elif "Frame" in line:
                frameMax = line.split()[-1]
        try:
            c = float(Concentration)
        except Exception:
            c = -1.0
        self.fConcentration = c
        lstHeader = [
            "%s %s" % (hdr, Comments),
            "%s Sample c= %s mg/ml" % (hdr, Concentration), hdr
        ]
        if frameMax is not None:
            lstHeader.append(hdr +
                             " Number of frames collected: %s" % frameMax)
        if exposureTime is not None:
            lstHeader.append(hdr +
                             " Exposure time per frame: %s" % exposureTime)

        if self.strRadiationDamage is None:
            lstHeader.append(
                "%s No significant radiation damage detected, merging %i files"
                % (hdr, len(self.lstMerged)))
        else:
            lstHeader.append("%s WARNING: %s" % (hdr, self.strRadiationDamage))
        lstHeader += [
            hdr + " Merged from: %s" % i.path.value for i in self.lstMerged
        ]
        if self.lstSub:
            lstHeader.append(hdr)
            lstHeader += ["%s %s" % (hdr, i) for i in self.lstSub]
        lstHeader += [hdr, hdr + " Sample Information:"]
        if storageTemperature is not None:
            lstHeader.append(hdr + " Storage Temperature (degrees C): %s" %
                             storageTemperature)
        if measurementTemperature is not None:
            lstHeader.append(hdr + " Measurement Temperature (degrees C): %s" %
                             measurementTemperature)

        lstHeader += [
            hdr + " Concentration: %s" % Concentration,
            "# Code: %s" % Code
        ]
        if infile is None:
            infile = self.strMergedFile
        if output is None:
            output = self.strMergedFile
        data = linesep.join(
            [i.strip() for i in open(infile) if not i.startswith(hdr)])
        with open(output, "w") as outfile:
            outfile.write(linesep.join(lstHeader))
            if not data.startswith(linesep):
                outfile.write(linesep)
            outfile.write(data)

    def doSuccessExecWait(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecWait")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsSmartMergev1_6.doSuccessExecWait")
        self.retrieveMessages(_edPlugin)
        xsdo = _edPlugin.dataOutput
        self.DEBUG("ExecWait Output:%s" % xsdo.marshal())
        if (xsdo.timedOut is not None) and bool(xsdo.timedOut.value):
            strErr = "Error in waiting for all input files to arrive"
            self.addExecutiveSummaryLine(
                "EDPluginBioSaxsSmartMergev1_6.doSuccessExecWait :" + strErr)
            self.ERROR(strErr)
            self.setFailure()

    def doFailureExecWait(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecWait")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureExecWait")
        self.retrieveMessages(_edPlugin)
        strErr = "Error in waiting for all input files to arrive"
        self.ERROR(strErr)
        self.addExecutiveSummaryLine(strErr)
        self.setFailure()

    def doSuccessExecDataver(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecDataver")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecDataver")
        self.retrieveMessages(_edPlugin)
        self.rewriteHeader(_edPlugin.dataOutput.outputCurve.path.value,
                           output=self.strMergedFile)

    def doFailureExecDataver(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecDataver")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureExecDataver")
        self.retrieveMessages(_edPlugin)
        strErr = "Error in Processing of Atsas 'dataver'"
        self.addExecutiveSummaryLine(strErr)
        self.ERROR(strErr)
        self.setFailure()

    def datcmp(self, file1, file2):
        #def doSuccessExecDatcmp(self, _edPlugin=None):
        #         self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecDatcmp")
        #         self.retrieveSuccessMessages(_edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecDatcmp")
        #         self.retrieveMessages(_edPlugin)
        data1 = loadtxt(file1.path.value)
        data2 = loadtxt(file2.path.value)
        with self.locked():
            try:
                cor = gof(data1, data2)
                fidelity = cor.P
            except Exception as err:
                strErr = "No fidelity in output of datcmp"
                self.error(strErr)
                self.addExecutiveSummaryLine(strErr)
                self.addExecutiveSummaryLine(str(err))
                #self.lstExecutiveSummary.append(strErr)
                #self.lstExecutiveSummary.append(err)
                if not "maximum recursion depth exceeded" in str(err):
                    self.setFailure()
                fidelity = 0
            lstIdx = [
                self.lstStrInput.index(file1.path.value),
                self.lstStrInput.index(file2.path.value)
            ]
            lstIdx.sort()
            self.dictSimilarities[tuple(lstIdx)] = fidelity
            lstIdx.reverse()
            self.dictSimilarities[tuple(lstIdx)] = fidelity
            if fidelity <= 0:
                logFid = "infinity"
            else:
                logFid = "%.2f" % (-log(fidelity, 10))
            self.addExecutiveSummaryLine(
                "-log(Fidelity) between %s and %s is %s" %
                (os.path.basename(file1.path.value),
                 os.path.basename(file2.path.value), logFid))

#     def doFailureExecDatcmp(self, _edPlugin=None):
#         self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecDatcmp")
#         self.retrieveFailureMessages(_edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureExecDatcmp")
#         self.retrieveMessages(_edPlugin)
#         self.addExecutiveSummaryLine("Failure in Processing of Atsas 'datcmp'")
#         self.setFailure()

    def doSuccessExecAutoSub(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecAutoSub")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecAutoSub")
        self.retrieveMessages(_edPlugin)
        self.autoRg = _edPlugin.dataOutput.autoRg
        if _edPlugin.dataOutput.subtractedCurve is not None:
            subcurve = _edPlugin.dataOutput.subtractedCurve
            if os.path.exists(subcurve.path.value):
                self.strSubFile = subcurve.path.value
        self.xsBestBuffer = _edPlugin.dataOutput.bestBuffer
        self.bestBufferType = _edPlugin.dataOutput.bestBufferType.value
        if self.bestBufferType == 'average':
            self.bufferFrames = self.lstMerged
            #if self.__class__.dictFrames[self.__class__.lastBuffer] is not None:
            #	if self.__class__.dictFrames[self.__class__.lastBuffer]['averaged'] is not None:
            #	    self.bufferFrames = self.bufferFrames + self.__class__.dictFrames[self.__class__.lastBuffer]['averaged']
            self.bufferFrames = self.bufferFrames + self.getAveragedFrameByFilename(
                self.__class__.lastBuffer)

        elif self.bestBufferType == self.__class__.lastBuffer:
            #if self.__class__.dictFrames[self.__class__.lastBuffer] is not None:
            #    if self.__class__.dictFrames[self.__class__.lastBuffer]['averaged'] is not None:
            #        self.bufferFrames = self.__class__.dictFrames[self.__class__.lastBuffer]['averaged']
            self.bufferFrames = self.getAveragedFrameByFilename(
                self.__class__.lastBuffer)
        else:
            self.bufferFrames = self.lstMerged
        self.addExecutiveSummaryLine(
            _edPlugin.dataOutput.status.executiveSummary.value)

    def doFailureExecAutoSub(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecAutoSub")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doFailureExecAutoSub")
        self.retrieveMessages(_edPlugin)
        strErr = "Error in Processing of EDNA 'AutoSub'"
        # clean up "EDNA memory"
        self.__class__.lastBuffer = self.dataInput.mergedCurve
        self.__class__.lastSample = None
        self.ERROR(strErr)
        self.addExecutiveSummaryLine(strErr)
        self.setFailure()

    def doSuccessSaxsAnalysis(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessSaxsAnalysis")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessSaxsAnalysis")
        self.retrieveMessages(_edPlugin)
        self.gnom = _edPlugin.dataOutput.gnom
        self.volume = _edPlugin.dataOutput.volume
        self.rti = _edPlugin.dataOutput.rti
        self.xsScatterPlot = _edPlugin.dataOutput.scatterPlot
        self.xsGuinierPlot = _edPlugin.dataOutput.guinierPlot
        self.xsKratkyPlot = _edPlugin.dataOutput.kratkyPlot
        if hasattr(_edPlugin.dataOutput, 'kratkyRgPlot'):
            self.xsKratkyRgPlot = _edPlugin.dataOutput.kratkyRgPlot
        if hasattr(_edPlugin.dataOutput, 'kratkyVcPlot'):
            self.xsKratkyVcPlot = _edPlugin.dataOutput.kratkyVcPlot

        self.xsDensityPlot = _edPlugin.dataOutput.densityPlot
        self.addExecutiveSummaryLine(
            _edPlugin.dataOutput.status.executiveSummary.value)

    def doFailureSaxsAnalysis(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureSaxsAnalysis")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureSaxsAnalysis")
        self.retrieveMessages(_edPlugin)
        strErr = "Error in Processing of EDNA SaxsAnalysis = AutoRg => datGnom => datPorod"
        self.ERROR(strErr)
        self.addExecutiveSummaryLine(strErr)
        self.setFailure()

    def doSuccessISPyB(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessISPyB")
        self.addExecutiveSummaryLine("Registered in ISPyB")
        self.retrieveMessages(_edPlugin)

    def doFailureISPyB(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureISPyB")
        self.addExecutiveSummaryLine("Failed to registered in ISPyB")
        self.retrieveMessages(_edPlugin)

    def doSuccessExecRsync(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecRsync")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecRsync")
        self.retrieveMessages(_edPlugin)
        self.gnom.gnomFile = XSDataFile(
            XSDataString(
                os.path.join(self.outdir,
                             os.path.split(self.gnom.gnomFile.path.value)[1])))

    def doFailureExecRsync(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecRsync")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doFailureExecRsync")
        self.retrieveMessages(_edPlugin)
        self.setFailure()
Exemplo n.º 23
0
class EDPluginBioSaxsHPLCv1_0(EDPluginControl):
    """
    plugin for processing Saxs data coming from HPLC

    runs subsequently:
    *ProcessOneFile,
    *subtraction of buffer
    *SaxsAnalysis

    todo:
    only store references: Wait for flush to construct HDF5 file and (possibly) web pages with PNG graphs
    """

    strControlledPluginProcessOneFile = "EDPluginBioSaxsProcessOneFilev1_3"
    strControlledPluginDatop = "EDPluginExecDatopv1_0"
    strControlledPluginSaxsAnalysis = "EDPluginControlSaxsAnalysisv1_0"
    strControlledPluginDatCmp = "EDPluginExecDatcmpv1_0"
    strControlledPluginDatAver = "EDPluginExecDataverv1_0"
    dictHPLC = {}  #key=runId, value= HPLCrun instance
    _sem = Semaphore()

    def __init__(self):
        """
        """
        EDPluginControl.__init__(self)
        self.setXSDataInputClass(XSDataInputBioSaxsHPLCv1_0)
        self.__edPluginProcessOneFile = None
        self.__edPluginSubtract = None
        self.__edPluginSaxsAnalysis = None
        self.__edPluginDatCmp = None
        self.xsDataResult = XSDataResultBioSaxsHPLCv1_0()
        self.runId = None
        self.frameId = None
        self.frame = None
        self.hplc_run = None
        self.curve = None
        self.subtracted = None
        self.lstExecutiveSummary = []
        self.isBuffer = False

    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.checkParameters")
        self.checkMandatoryParameters(self.dataInput, "Data Input is None")
        self.checkMandatoryParameters(self.dataInput.rawImage, "No raw image")
        self.checkMandatoryParameters(self.dataInput.sample,
                                      "no Sample parameter")
        self.checkMandatoryParameters(self.dataInput.experimentSetup,
                                      "No experimental setup parameter")

    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.preProcess")
        sdi = self.dataInput
        if sdi.runId is not None:
            self.runId = sdi.runId.value
        else:
            path = sdi.rawImage.path.value
            if "_" in path:
                self.runId = path[::-1].split("_", 1)[1][::-1]
            else:
                self.runId = path
        with self._sem:
            if self.runId not in self.dictHPLC:
                self.dictHPLC[self.runId] = HPLCrun(self.runId)
        self.hplc_run = self.dictHPLC[self.runId]
        if sdi.frameId is not None:
            self.frameId = sdi.frameId.value
        else:
            path = sdi.rawImage.path.value
            if "_" in path:
                digits = os.path.splitext(
                    os.path.basename(path))[0].split("_")[-1]
                try:
                    self.frameId = int(digits)
                except ValueError:
                    self.WARNING(
                        "frameId is supposed to be an integer, I got %s" %
                        digits)
                    self.frameId = digits
            else:
                self.warning("using frameID=0 in tests, only")
                self.frameId = 0
        with self._sem:
            self.frame = HPLCframe(self.runId, self.frameId)
            self.hplc_run.frames[self.frameId] = self.frame

        if sdi.bufferCurve and os.path.exists(sdi.bufferCurve.path.value):
            with self._sem:
                self.hplc_run.buffer = sdi.bufferCurve.path.value

        if self.hplc_run.hdf5_filename:
            hplc = self.hplc_run.hdf5_filename
        elif sdi.hplcFile:
            hplc = sdi.hplcFile.path.value
        else:
            path = sdi.rawImage.path.value
            if "_" in path:
                hplc = "_".join(
                    os.path.splitext(path)[0].split("_")[:-1]) + ".h5"
            else:
                hplc = os.path.splitext(path)[0] + ".h5"

        if not self.hplc_run.hdf5_filename:
            with self._sem:
                self.hplc_run.init_hdf5(hplc)
#        self.xsDataResult.hplcFile = XSDataFile(XSDataString(hplc))

    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.process")

        xsdIn = XSDataInputBioSaxsProcessOneFilev1_0(
            rawImage=self.dataInput.rawImage,
            sample=self.dataInput.sample,
            experimentSetup=self.dataInput.experimentSetup,
            rawImageSize=self.dataInput.rawImageSize,
            normalizedImage=self.dataInput.normalizedImage,
            integratedCurve=self.dataInput.integratedCurve,
            runId=self.dataInput.runId,
            frameId=self.dataInput.frameId)
        self.__edPluginProcessOneFile = self.loadPlugin(
            self.strControlledPluginProcessOneFile)
        self.__edPluginProcessOneFile.dataInput = xsdIn
        self.__edPluginProcessOneFile.connectSUCCESS(
            self.doSuccessProcessOneFile)
        self.__edPluginProcessOneFile.connectFAILURE(
            self.doFailureProcessOneFile)
        self.__edPluginProcessOneFile.executeSynchronous()

        if self.isFailure():
            return

        xsdIn = XSDataInputDatcmp(inputCurve=[
            XSDataFile(XSDataString(self.hplc_run.first_curve)),
            XSDataFile(XSDataString(self.curve))
        ])
        self.__edPluginDatCmp = self.loadPlugin(self.strControlledPluginDatCmp)
        self.__edPluginDatCmp.dataInput = xsdIn
        self.__edPluginDatCmp.connectSUCCESS(self.doSuccessDatCmp)
        self.__edPluginDatCmp.connectFAILURE(self.doFailureDatCmp)
        self.__edPluginDatCmp.executeSynchronous()

        if self.isFailure() or self.isBuffer:
            return

        if self.dataInput.subtractedCurve is not None:
            subtracted = self.dataInput.subtractedCurve.path.value
        else:
            subtracted = os.path.splitext(self.curve)[0] + "_sub.dat"
        xsdIn = XSDataInputDatop(inputCurve=[
            XSDataFile(XSDataString(self.curve)),
            XSDataFile(XSDataString(self.hplc_run.first_curve))
        ],
                                 outputCurve=XSDataFile(
                                     XSDataString(subtracted)),
                                 operation=XSDataString("sub"))
        self.__edPluginDatop = self.loadPlugin(self.strControlledPluginDatop)
        self.__edPluginDatop.dataInput = xsdIn
        self.__edPluginDatop.connectSUCCESS(self.doSuccessDatop)
        self.__edPluginDatop.connectFAILURE(self.doFailureDatop)
        self.__edPluginDatop.executeSynchronous()

        if self.subtracted and os.path.exists(self.subtracted):
            xsdIn = XSDataInputSaxsAnalysis(
                scatterCurve=XSDataFile(XSDataString(self.subtracted)),
                gnomFile=XSDataFile(
                    XSDataString(
                        os.path.splitext(self.subtracted)[0] + ".out")))
            self.__edPluginSaxsAnalysis = self.loadPlugin(
                self.strControlledPluginSaxsAnalysis)
            self.__edPluginSaxsAnalysis.dataInput = xsdIn
            self.__edPluginSaxsAnalysis.connectSUCCESS(
                self.doSuccessSaxsAnalysis)
            self.__edPluginSaxsAnalysis.connectFAILURE(
                self.doFailureSaxsAnalysis)
            self.__edPluginSaxsAnalysis.executeSynchronous()

    def postProcess(self, _edObject=None):
        EDPluginControl.postProcess(self)
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.postProcess")
        if self.hplc_run.buffer:
            self.xsDataResult.bufferCurve = XSDataFile(
                XSDataString(self.hplc_run.buffer))
        if self.curve:
            self.xsDataResult.integratedCurve = XSDataFile(
                XSDataString(self.curve))
        if self.subtracted:
            self.xsDataResult.subtractedCurve = XSDataFile(
                XSDataString(self.subtracted))

    def finallyProcess(self, _edObject=None):
        EDPluginControl.finallyProcess(self)
        executiveSummary = os.linesep.join(self.lstExecutiveSummary)
        self.xsDataResult.status = XSDataStatus(
            executiveSummary=XSDataString(executiveSummary))
        self.dataOutput = self.xsDataResult
        if self.frame:
            self.frame.processing = False

    def average_buffers(self):
        """
        Average out all buffers
        """
        self.lstExecutiveSummary.append("Averaging out buffer files: " +
                                        ", ".join(self.hplc_run.for_buffer))
        xsdIn = XSDataInputDataver(inputCurve=[
            XSDataFile(XSDataString(i)) for i in self.hplc_run.for_buffer
        ])
        if self.dataInput.bufferCurve:
            xsdIn.outputCurve = self.dataInput.bufferCurve
        else:
            xsdIn.outputCurve = XSDataFile(
                XSDataString(
                    self.hplc_run.first_curve[::-1].split("_", 1)[1][::-1] +
                    "_buffer_aver%02i.dat" % len(self.hplc_run.for_buffer)))
        self.__edPluginDatAver = self.loadPlugin(
            self.strControlledPluginDatAver)
        self.__edPluginDatAver.dataInput = xsdIn
        self.__edPluginDatAver.connectSUCCESS(self.doSuccessDatAver)
        self.__edPluginDatAver.connectFAILURE(self.doFailureDatAver)
        self.__edPluginDatAver.executeSynchronous()

    def doSuccessProcessOneFile(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessProcessOneFile")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessProcessOneFile")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        output = _edPlugin.dataOutput
        if not output.integratedCurve:
            strErr = "Edna plugin ProcessOneFile did not produce integrated curve"
            self.ERROR(strErr)
            self.lstExecutiveSummary.append(strErr)
            self.setFailure()
            return
        self.curve = output.integratedCurve.path.value
        if not os.path.exists(self.curve):
            strErr = "Edna plugin ProcessOneFile: integrated curve not on disk !!"
            self.ERROR(strErr)
            self.lstExecutiveSummary.append(strErr)
            self.setFailure()
            return
        self.xsDataResult.integratedCurve = output.integratedCurve
        self.xsDataResult.normalizedImage = output.normalizedImage
        if output.experimentSetup and output.experimentSetup.timeOfFrame:
            startTime = output.experimentSetup.timeOfFrame.value
        else:
            startTime = None
        with self._sem:
            if not self.hplc_run.first_curve:
                self.hplc_run.first_curve = self.curve
                self.hplc_run.start_time = startTime
        self.frame.curve = self.curve
        self.frame.time = startTime

    def doFailureProcessOneFile(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureProcessOneFile")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureProcessOneFile")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        else:
            self.lstExecutiveSummary.append(
                "Edna plugin ProcessOneFile failed.")
        self.setFailure()

    def doSuccessDatop(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessDatop")
        self.retrieveSuccessMessages(_edPlugin,
                                     "EDPluginBioSaxsHPLCv1_0.doSuccessDatop")
        if _edPlugin and _edPlugin.dataOutput:
            output = _edPlugin.dataOutput
            if output.status and output.status.executiveSummary:
                self.lstExecutiveSummary.append(
                    output.status.executiveSummary.value)
            if output.outputCurve:
                self.subtracted = output.outputCurve.path.value
                if os.path.exists(self.subtracted):
                    self.xsDataResult.subtractedCurve = output.outputCurve
                    self.frame.subtracted = self.subtracted
                else:
                    strErr = "Edna plugin datop did not produce subtracted file %s" % self.subtracted
                    self.ERROR(strErr)
                    self.lstExecutiveSummary.append(strErr)
                    self.setFailure()

    def doFailureDatop(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureDatop")
        self.retrieveFailureMessages(_edPlugin,
                                     "EDPluginBioSaxsHPLCv1_0.doFailureDatop")
        strErr = "Edna plugin datop failed."
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        else:
            self.lstExecutiveSummary.append(strErr)
        self.ERROR(strErr)
        self.setFailure()

    def doSuccessSaxsAnalysis(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessSaxsAnalysis")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessSaxsAnalysis")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        gnom = _edPlugin.dataOutput.gnom
        if gnom:
            if gnom.rgGnom:
                self.frame.gnom = gnom.rgGnom.value
            if gnom.dmax:
                self.frame.Dmax = gnom.dmax.value
            if gnom.total:
                self.frame.total = gnom.total.value
            self.xsDataResult.gnom = gnom

        volume = _edPlugin.dataOutput.volume
        if volume:
            self.frame.volume = volume.value
            self.xsDataResult.volume = volume
        rg = _edPlugin.dataOutput.autoRg
        if rg:
            if rg.rg:
                self.frame.Rg = rg.rg.value
            if rg.rgStdev:
                self.frame.Rg_Stdev = rg.rgStdev.value
            if rg.i0:
                self.frame.I0 = rg.i0.value
            if rg.i0Stdev:
                self.frame.I0_Stdev = rg.i0Stdev.value
            if rg.quality:
                self.frame.quality = rg.quality.value
            self.xsDataResult.autoRg = rg

    def doFailureSaxsAnalysis(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureSaxsAnalysis")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureSaxsAnalysis")
        strErr = "Edna plugin SaxsAnalysis failed."
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
            self.lstExecutiveSummary.append(strErr)
        else:
            self.lstExecutiveSummary.append(strErr)
        self.setFailure()

    def doSuccessDatCmp(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessDatCmp")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessDatCmp")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.fidelity:
            fidelity = _edPlugin.dataOutput.fidelity.value
        else:
            strErr = "No fidelity in output of datcmp"
            self.error(strErr)
            self.lstExecutiveSummary.append(strErr)
            #self.setFailure()
            fidelity = 0
        if self.hplc_run.buffer is None:
            if fidelity > 0:
                self.isBuffer = True
                if fidelity > 0.1:
                    self.hplc_run.for_buffer.append(self.curve)
            else:
                self.average_buffers()
        elif fidelity > 0:
            self.isBuffer = True

    def doFailureDatCmp(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureDatCmp")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureDatCmp")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        else:
            self.lstExecutiveSummary.append("Edna plugin DatCmp failed.")
        self.setFailure()

    def doSuccessDatAver(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessDatAver")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessDatAver")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.outputCurve:
            bufferCurve = _edPlugin.dataOutput.outputCurve.path.value
            if os.path.exists(bufferCurve):
                with self._sem:
                    self.hplc_run.buffer = bufferCurve
            else:
                strErr = "DatAver claimed buffer is in %s but no such file !!!" % bufferCurve
                self.ERROR(strErr)
                self.lstExecutiveSummary.append(strErr)
                self.setFailure()

    def doFailureDatAver(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureDatAver")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureDatAver")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        else:
            self.lstExecutiveSummary.append("Edna plugin DatAver failed.")
        self.setFailure()
Exemplo n.º 24
0
class EDObject(object):
    """
    Virtual base class for all EDNA Objects (classes).
    It offers some synchronization and locking capabilities to make the code thread safe.
    """
    __semaphoreId = Semaphore()
    __iId_class = 0

    def __init__(self):
        """
        Constructor of the main pure virtual class.
        This constructor implements:
        - the creation of the semaphore
        - definition of timer object (uninitialized as potentially not used)
        """
        object.__init__(self)
        with self.__class__.__semaphoreId:
            self.__class__.__iId_class += 1
            self.__iId = self.__class__.__iId_class
        self.__semaphore = Semaphore()
        self.__fTimeInit = None
        self.__fTimeEnd = None
        self.__classname = None

    def getId(self):
        return self.__iId

    def getClassName(self):
        """
        Retrieves the name of the class
        @return: the name of the class 
        @rtype: string 
        """
        return self.__class__.__name__

    def synchronizeOn(self):
        """
        This method must be used in together with the method synchronizeOff().
        This method makes the code threadsafe till the method synchronizeOff
        is called.
        """
        self.__semaphore.acquire()

    def synchronizeOff(self):
        """
        This method must be used in together with the method synchronizeOn().
        """
        self.__semaphore.release()

    def locked(self):
        return self.__semaphore

    def setTimeInit(self):
        """
        Initializes the timer for the object
        """
        if self.__fTimeInit is None:
            self.__fTimeInit = time.time()

    def getTimeInit(self):
        """
        Retrieves the time of initialization
        @return: number of seconds since epoch
        @rtype: float  
        """
        return self.__fTimeInit

    def setTimeEnd(self):
        """
        Set the end of calculation time for the given object
        """
        if self.__fTimeEnd is None:
            self.__fTimeEnd = time.time()

    def getTimeEnd(self):
        """
        Retrieves the time of end of task
        @return: number of seconds since epoch
        @rtype: float  
        """
        return self.__fTimeEnd

    def getRunTime(self):
        """
        @returns: the RunTime for the given object
        @rtype: float
        """
        fRetrunRunTime = 0.0
        if self.__fTimeInit is not None:
            if self.__fTimeEnd is None:
                fRetrunRunTime = time.time() - self.__fTimeInit
            else:
                fRetrunRunTime = self.__fTimeEnd - self.__fTimeInit
        return fRetrunRunTime
Exemplo n.º 25
0
class EDPluginExecPyFAIv1_0(EDPluginExec):
    """
    This is the basic plugin of PyFAI for azimuthal integration
    """

    _dictGeo = {} #key:tuple(ai.param), value= ai
    _sem = Semaphore()
    def __init__(self):
        """
        """
        EDPluginExec.__init__(self)
        self.setXSDataInputClass(XSDataInputPyFAI)
        self.shared = None
        self.strOutputFile = None
        self.ai = None #this is the azimuthal integrator to use
        self.data = None
        self.mask = None
        self.nbPt = None
        self.dummy = None
        self.delta_dummy = None
        self.npaOut = None

    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginExecPyFAIv1_0.checkParameters")
        self.checkMandatoryParameters(self.dataInput, "Data Input is None")
#        self.checkMandatoryParameters(self.dataInput.geometry, "No Geometry given")
#        self.checkMandatoryParameters(self.dataInput.detector, "No Detector given")
        self.checkMandatoryParameters(self.dataInput.input, "No input image given")
        self.checkMandatoryParameters(self.dataInput.nbPt, "No Number of points given for integration")

    @staticmethod
    def getDetector(xsDetector):
        detector = None
        if xsDetector.name and (xsDetector.name.value in dir(pyFAI.detectors)):
            detector = getattr(pyFAI.detectors, xsDetector.name.value)()
        else:
            pixel2 = EDUtilsUnit.getSIValue(xsDetector.pixelSizeX)
            pixel1 = EDUtilsUnit.getSIValue(xsDetector.pixelSizeY)
            if xsDetector.splineFile and os.path.isFile(xsDetector.splineFile.path.value):
                dictGeo = {"pixel1":pixel1, "pixel2":pixel2, "splineFile":xsDetector.splineFile.path.value}
            else:
                dictGeo = {"pixel1":pixel1, "pixel2":pixel2, "splineFile":None}
            detector = pyFAI.detectors.Detector()
            detector.setPyFAI(**dictGeo)
        return detector

    def preProcess(self, _edObject=None):
        EDPluginExec.preProcess(self)
        self.DEBUG("EDPluginExecPyFAIv1_0.preProcess")
        sdi = self.dataInput
        ai = pyFAI.AzimuthalIntegrator()
        if sdi.geometryFit2D is not None:
            xsGeometry = sdi.geometryFit2D
            detector = self.getDetector(xsGeometry.detector)
            d = {"direct": EDUtilsUnit.getSIValue(xsGeometry.distance) * 1000, #fit2D takes the distance in mm
               "centerX": xsGeometry.beamCentreInPixelsX.value ,
               "centerY":xsGeometry.beamCentreInPixelsY.value  ,
               "tilt": xsGeometry.angleOfTilt.value,
               "tiltPlanRotation": xsGeometry.tiltRotation.value}
            d.update(detector.getFit2D())
            ai.setFit2D(**d)
        elif sdi.geometryPyFAI is not None:
            xsGeometry = sdi.geometryPyFAI
            detector = self.getDetector(xsGeometry.detector)
            d = {"dist": EDUtilsUnit.getSIValue(xsGeometry.sampleDetectorDistance),
               "poni1": EDUtilsUnit.getSIValue(xsGeometry.pointOfNormalIncidence1),
               "poni2": EDUtilsUnit.getSIValue(xsGeometry.pointOfNormalIncidence2),
               "rot1": EDUtilsUnit.getSIValue(xsGeometry.rotation1),
               "rot2": EDUtilsUnit.getSIValue(xsGeometry.rotation2),
               "rot3": EDUtilsUnit.getSIValue(xsGeometry.rotation3)}
            d.update(detector.getPyFAI())
            ai.setPyFAI(**d)
        else:
            strError = "Geometry definition in %s, not recognized as a valid geometry%s %s" % (sdi, os.linesep, sdi.marshal())
            self.ERROR(strError)
            raise RuntimeError(strError)

        ########################################################################
        # Choose the azimuthal integrator
        ########################################################################

        with self.__class__._sem:
            if tuple(ai.param) in self.__class__._dictGeo:
                self.ai = self.__class__._dictGeo[tuple(ai.param)]
            else:
                self.__class__._dictGeo[tuple(ai.param)] = ai
                self.ai = ai

        self.data = EDUtilsArray.getArray(self.dataInput.input).astype(float)
        if sdi.dark is not None:
            self.data -= EDUtilsArray.getArray(sdi.dark)
        if sdi.flat is not None:
            self.data /= EDUtilsArray.getArray(sdi.flat)
        if sdi.mask is not None:
            self.mask = EDUtilsArray.getArray(sdi.mask)
        if sdi.wavelength is not None:
            self.ai.wavelength = EDUtilsUnit.getSIValue(sdi.wavelength)
        if sdi.output is not None:
            self.strOutputFile = sdi.output.path.value
        if sdi.dummy is not None:
            self.dummy = sdi.dummy.value
        if sdi.deltaDummy is not None:
            self.delta_dummy = sdi.deltaDummy.value
        if sdi.nbPt:
            self.nbPt = sdi.nbPt.value

    def process(self, _edObject=None):
        EDPluginExec.process(self)
        self.DEBUG("EDPluginExecPyFAIv1_0.process")
        data = EDUtilsArray.getArray(self.dataInput.input)
        if self.dataInput.saxsWaxs and self.dataInput.saxsWaxs.value.lower().startswith("s"):
            out = self.ai.saxs(self.data,
                               nbPt=self.nbPt,
                               filename=self.strOutputFile,
                               mask=self.mask,
                               dummy=self.dummy,
                               delta_dummy=self.delta_dummy)
        else:
            out = self.ai.xrpd(self.data,
                               nbPt=self.nbPt,
                               filename=self.strOutputFile,
                               mask=self.mask,
                               dummy=self.dummy,
                               delta_dummy=self.delta_dummy)
        self.npaOut = np.hstack((i.reshape(-1, 1) for i in out if i is not None))


    def postProcess(self, _edObject=None):
        EDPluginExec.postProcess(self)
        self.DEBUG("EDPluginExecPyFAIv1_0.postProcess")
        # Create some output data
        if self.strOutputFile:
            output = XSDataImageExt(path=XSDataString(self.strOutputFile))
        elif self.shared:
            output = XSDataImageExt(shared=XSDataString(self.shared))
        else:
            output = XSDataImageExt(array=EDUtilsArray.arrayToXSData(self.npaOut))
        xsDataResult = XSDataResultPyFAI(output=output)

        self.setDataOutput(xsDataResult)
Exemplo n.º 26
0
class EDPluginWaitFile(EDPlugin):
    """
    Plugins that waits for a file to be written and reach a certain size
    """
    #offers some extra seconds between officiel timeout and allows the plugin to finish gracefully.
    EXTRA_TIME = 5
    DELTA_TIME = 1
    DEFAULT_TIMEOUT = 2
    config_timeout = None
    writeXmlInOut = True
    writeDataXMLOutput = True
    writeDataXMLInput = True
    sem = Semaphore()

    def __init__(self):
        """
        """
        EDPlugin.__init__(self)
        self.setXSDataInputClass(XSDataInputWaitFile)
        self.__filename = None
        self.__exists = None
        self.__filesize = None
        self.__expectedSize = None
        self.__timeout = self.DEFAULT_TIMEOUT

    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginWaitFile.checkParameters")
        self.checkMandatoryParameters(self.getDataInput(),
                                      "Data Input is None")
        self.checkMandatoryParameters(self.getDataInput().getExpectedFile(),
                                      "Data Input is None")
        self.checkMandatoryParameters(self.getDataInput().getExpectedSize(),
                                      "Data Input is None")

    def configure(self):
        """

        Configuration step of the plugin: mainly extend the timeout by 5 seconds to let the plugin finish.

        """
        if self.__class__.config_timeout is None:
            with self.__class__.sem:
                if self.__class__.config_timeout is None:
                    iTimeOut = self.config.get(EDPlugin.CONF_TIME_OUT, None)
                    if iTimeOut is not None:
                        self.DEBUG(
                            "EDPlugin.configure: Setting time out to %d s from plugin configuration."
                            % iTimeOut)
                        self.__class__.config_timeout = iTimeOut
                    else:
                        self.__class__.config_timeout = self.getDefaultTimeOut(
                        )
                    self.__class__.writeXmlInOut = bool(
                        self.config.get(self.CONF_WRITE_XML_INPUT_OUTPUT,
                                        True))
                    self.__class__.writeDataXMLOutput = bool(
                        self.config.get(self.CONF_WRITE_XML_OUTPUT, True))
                    self.__class__.writeDataXMLInput = bool(
                        self.config.get(self.CONF_WRITE_XML_INPUT, True))
        self.__timeout = self.__class__.config_timeout
        self.setTimeOut(self.__class__.config_timeout +
                        EDPluginWaitFile.EXTRA_TIME)
        self.setWriteXMLInputOutput(self.writeXmlInOut)
        self.setWriteXMLOutput(self.writeDataXMLOutput)
        self.setWriteXMLInput(self.writeDataXMLInput)

    def preProcess(self, _edObject=None):
        EDPlugin.preProcess(self)
        self.DEBUG("EDPluginWaitFile.preProcess")
        self.__filename = self.getDataInput().getExpectedFile().getPath(
        ).getValue()
        self.__expectedSize = self.getDataInput().getExpectedSize().getValue()
        if self.getDataInput().getTimeOut():
            self.__timeout = self.getDataInput().getTimeOut().getValue()
            self.setTimeOut(self.getDataInput().getTimeOut().getValue() +
                            EDPluginWaitFile.EXTRA_TIME)

    def process(self, _edObject=None):
        EDPlugin.process(self)
        self.DEBUG("EDPluginWaitFile.process")
        self.DEBUG(
            "EDPluginWaitFile Plugin TimeOut is set to: %s, internal TimeOut is %s"
            % (self.getTimeOut(), self.__timeout))
        self.setTimeInit()
        dirname = os.path.dirname(self.__filename)
        while self.getRunTime() < self.__timeout:
            if os.path.isdir(dirname):
                fd = os.open(dirname, os.O_RDONLY)
                os.fstat(fd)
                os.close(fd)
                if os.path.exists(self.__filename):
                    self.__filesize = os.path.getsize(self.__filename)
                    if self.__filesize >= self.__expectedSize:
                        break
            time.sleep(EDPluginWaitFile.DELTA_TIME)
        self.setTimeEnd()

    def postProcess(self, _edObject=None):
        EDPlugin.postProcess(self)
        self.DEBUG("EDPluginWaitFile.postProcess: Waited for %.3f s" %
                   self.getRunTime())
        xsDataResult = XSDataResultWaitFile()
        if os.path.exists(self.__filename):
            xsDataFile = XSDataFile()
            xsDataFile.setPath(XSDataString(self.__filename))
            xsDataResult.setActualFile(xsDataFile)
            xsDataResult.setActualSize(
                XSDataInteger(os.path.getsize(self.__filename)))
        xsDataResult.setTimedOut(
            XSDataBoolean(self.getRunTime() >= self.__timeout))
        # Create some output data
        self.setDataOutput(xsDataResult)
Exemplo n.º 27
0
class Reprocess(object):
    EDNAPluginName = "EDPluginBioSaxsProcessOneFilev1_4"
    hc = 12.398419292004204
    def __init__(self):
        self.XML = "<XSDataInput>\
<normalizedImage><path><value>${FULLPATH}</value></path></normalizedImage>\
<correctedImage><path><value>${DIRDIRNAME}/2d/${BASENAME}.edf</value></path></correctedImage>\
<normalizedImageSize><value>4100000</value></normalizedImageSize>\
<integratedCurve><path><value>${DIRDIRNAME}/1d/${BASENAME}.edf</value></path></integratedCurve>\
<maskFile><path><value>${MASKFILE}</value></path></maskFile>\
<code><value>BSA</value></code>\
</XSDataInput>"
        self.maskfile = None
        self.dataFiles = []
        self.wavelength = 1.0
        self.debug = False
        self.mode = "offline"
        self.newerOnly = False
        self.nbcpu = multiprocessing.cpu_count()
        self.cpu_sem = Semaphore(self.nbcpu)
        self.process_sem = Semaphore()
        self.queue = Queue()
        
    def fileName2xml(self, filename):
        """Here we create the XML string to be passed to the EDNA plugin from the input filename
        This can / should be modified by the final user
        
        @param filename: full path of the input file
        @type filename: python string representing the path
        @rtype: XML string
        @return: python string  
        """
        if filename.endswith(".edf"):
            FULLPATH = os.path.abspath(filename)
            DIRNAME, NAME = os.path.split(FULLPATH)
            DIRDIRNAME = os.path.dirname(DIRNAME)
            BASENAME, EXT = os.path.splitext(NAME)
            if not os.path.isdir(os.path.join(DIRDIRNAME, "1d")):
                   os.makedirs(os.path.join(DIRDIRNAME, "1d"), int("775", 8))
            return self.xml.replace("${FULLPATH}", FULLPATH).\
                replace("${DIRNAME}", DIRNAME).replace("${NAME}", NAME).\
                replace("${DIRDIRNAME}", DIRDIRNAME).replace("${BASENAME}", BASENAME).\
                replace("${EXT}", EXT).replace("$MASKFILE", self.maskfile or "")

    def XMLerr(self, strXMLin):
        """
        This is an example of XMLerr function ... it prints only the name of the file created
        @param srXMLin: The XML string used to launch the job
        @type strXMLin: python string with the input XML
        @rtype: None
        @return: None     
        """
        self.cpu_sem.release()
        if type(strXMLin) not in types.StringTypes:
            strXMLin= strXMLin.marshal()
        EDVerbose.WARNING("Error in the processing of :\n%s" % strXMLin)
        return None

    def XMLsuccess(self, strXMLin):
        """
        This is an example of XMLerr function ... it prints only the name of the file created
        @param srXMLin: The XML string used to launch the job
        @type strXMLin: python string with the input XML
        @rtype: None
        @return: None     
        """
        self.cpu_sem.release()
#        EDVerbose.WARNING("Error in the processing of :\n%s" % strXMLin)
        return None


    def parse(self):
        """
        parse options from command line
        """
        parser = optparse.OptionParser()
        parser.add_option("-V", "--version", dest="version", action="store_true",
                          help="print version of the program and quit", metavar="FILE", default=False)
        parser.add_option("-v", "--verbose",
                          action="store_true", dest="debug", default=False,
                          help="switch to debug/verbose mode")
        parser.add_option("-m", "--mask", dest="mask",
                      help="file containing the mask (for image reconstruction)", default=None)
        parser.add_option("-M", "--mode", dest="mode",
                      help="Mode can be online/offline/all", default="offline")
        parser.add_option("-o", "--out", dest="output",
                      help="file for log", default=None)
        parser.add_option("-w", "--wavelength", dest="wavelength", type="float",
                      help="wavelength of the X-Ray beam in Angstrom", default=None)
        parser.add_option("-e", "--energy", dest="energy", type="float",
                      help="energy of the X-Ray beam in keV (hc=%skeV.A)" % self.hc, default=None)
        parser.add_option("-t", "--template", dest="template", type="str",
                      help="template XML file", default=None)
        parser.add_option("-n", "--nbcpu", dest="nbcpu", type="int",
                      help="template XML file", default=self.nbcpu)


        (options, args) = parser.parse_args()

        # Analyse aruments and options
        if options.version:
            print("BioSaxs Azimuthal integration version %s" % __version__)
            sys.exit(0)
        if options.debug:
            EDVerbose.setVerboseDebugOn()
            self.debug = True
        if options.output:
            EDVerbose.setLogFileName(options.output)
        if options.mask and os.path.isfile(options.mask):
            self.maskfile = options.mask
        if options.template and os.path.isfile(options.template):
            self.xml = open(options.template).read()
        if options.wavelength:
            self.wavelength = 1e-10 * options.wavelength
        elif options.energy:
            self.wavelength = 1e-10 * self.hc / options.energy
        if options.mode=="offline":
            self.mode = "offline"
            self.newerOnly = False
        elif options.mode=="online":
            self.mode = "dirwarch"
            self.newerOnly = True
        elif options.mode=="dirwatch":
            self.mode = "dirwarch"
            self.newerOnly = False
        self.cpu_sem = Semaphore(options.nbcpu)
        self.nbcpu = options.nbcpu
        self.dataFiles = [f for f in args if os.path.isfile(f)]
        if not self.dataFiles:
            raise RuntimeError("Please provide datafiles or read the --help")


    def process(self):
        for fn in self.dataFiles:
            EDVerbose.screen("Processing file %s" % fn)
            edj = EDJob(self.EDNAPluginName)
            edj.dataInput = self.fileName2xml(fn)
            edj.connectSUCCESS(self.XMLsuccess)
            edj.connectFAILURE(self.XMLerr)
            self.queue.put(edj)
            if self.process_sem._Semaphore__value > 0 :
                t = threading.Thread(target=self.startProcessing)
                t.start()
        EDVerbose.screen("Back in main")
        while self.cpu_sem._Semaphore__value < self.nbcpu:
            time.sleep(0.1)
        EDJob.synchronizeAll()
        EDJob.stats()

    def startProcessing(self):
        with self.process_sem:
            while not self.queue.empty():
                self.cpu_sem.acquire()
                edj = self.queue.get()
                edj.execute()
Exemplo n.º 28
0
 def __init__(self, runId, first_curve=None):
     self.id = runId
     self.buffer = None
     self.first_curve = first_curve
     self.frames = {}  #key: id, value: HPLCframe instance
     self.curves = []
     self.for_buffer = []
     self.hdf5_filename = None
     self.hdf5 = None
     self.chunk_size = 250
     self.lock = Semaphore()
     if first_curve:
         self.files.append(first_curve)
     self.max_size = None
     self.start_time = None
     self.time = None
     self.gnom = None
     self.Dmax = None
     self.total = None
     self.volume = None
     self.Rg = None
     self.Rg_Stdev = None
     self.I0 = None
     self.I0_Stdev = None
     self.quality = None
     self.q = None
     self.size = None
     self.scattering_I = None
     self.scattering_Stdev = None
     self.subtracted_I = None
     self.subtracted_Stdev = None
     self.sum_I = None
     self.Vc = None
     self.Qr = None
     self.mass = None
     self.Vc_Stdev = None
     self.Qr_Stdev = None
     self.mass_Stdev = None
     self.buffer_frames = None
     self.merge_frames = None
     self.buffer_I = None
     self.buffer_Stdev = None
     self.merge_I = None
     self.merge_Stdev = None
     self.merge_curves = []
     self.merge_Rg = {}
     self.merge_analysis = {}
     self.merge_framesDIC = {}
     self.keys1d = [
         "gnom", "Dmax", "total", "volume", "Rg", "Rg_Stdev", "I0",
         "I0_Stdev", "quality", "sum_I", "Vc", "Qr", "mass", "Vc_Stdev",
         "Qr_Stdev", "mass_Stdev"
     ]
     self.keys2d = [
         "scattering_I", "scattering_Stdev", "subtracted_I",
         "subtracted_Stdev"
     ]
     self.keys_frames = ["buffer_frames", "merge_frames"]
     self.keys_merges = [
         "buffer_I", "buffer_Stdev", "merge_I", "merge_Stdev"
     ]
Exemplo n.º 29
0
class EDPluginExecMeasureOffsetv1_0(EDPluginExec):
    """
    An exec plugin that taked two images and measures the offset between the two.
    """
    __sem = Semaphore()

    CONF_CONVOLUTION = None
    CONF_CONVOLUTION_KEY = "convolution"
    CONF_CONVOLUTION_DEFAULT = "numpy"  #can be "fftw", "cuda" or simply falls back on numpy.fftpack

    def __init__(self):
        """
        """
        EDPluginExec.__init__(self)
        self.setXSDataInputClass(XSDataInputMeasureOffset)
        self.npaIm1 = None
        self.npaIm2 = None
        self.tOffset = None
        self.tCrop = None
        self.tCenter = None
        self.tWidth = None
        self.tSmooth = None
        self.bBackgroundsubtraction = False
        self.sobel = False

    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginExecMeasureOffsetv1_0.checkParameters")
        self.checkMandatoryParameters(self.dataInput, "Data Input is None")

    def configure(self):
        """
        Configures the plugin from the configuration file with the following parameters
         - The CONVOLUTION method
        """
        EDPluginExec.configure(self)
        if self.CONF_CONVOLUTION is None:
            with self.__class__.__sem:
                self.DEBUG("EDPluginExecMeasureOffsetv1_0.configure")
                xsPluginItem = self.getConfiguration()
                if xsPluginItem is None:
                    self.WARNING(
                        "EDPluginExecMeasureOffsetv1_0.configure: No plugin item defined."
                    )
                    xsPluginItem = XSPluginItem()
                strCONVOLUTION = EDConfiguration.getStringParamValue(
                    xsPluginItem, self.CONF_CONVOLUTION_KEY)
                if strCONVOLUTION is None:
                    self.WARNING("EDPluginExecMeasureOffsetv1_0.configure: No configuration parameter found for: %s using default value: %s\n%s"\
                                % (self.CONF_CONVOLUTION_KEY, self.CONF_CONVOLUTION_DEFAULT, xsPluginItem.marshal()))
                    self.__class__CONF_CONVOLUTION = self.CONF_CONVOLUTION_DEFAULT
                else:
                    self.__class__.CONF_CONVOLUTION = strCONVOLUTION.strip(
                    ).lower()

    def preProcess(self, _edObject=None):
        EDPluginExec.preProcess(self)
        self.DEBUG("EDPluginExecMeasureOffsetv1_0.preProcess")
        sdi = self.dataInput
        images = sdi.image
        arrays = sdi.array

        if len(images) == 2:
            self.npaIm1 = numpy.array(EDUtilsArray.getArray(images[0]))
            self.npaIm2 = numpy.array(EDUtilsArray.getArray(images[1]))
        elif len(arrays) == 2:
            self.npaIm1 = EDUtilsArray.xsDataToArray(arrays[0])
            self.npaIm2 = EDUtilsArray.xsDataToArray(arrays[1])
        else:
            strError = "EDPluginExecMeasureOffsetv1_0.preProcess: You should either provide two images or two arrays, but I got: %s" % sdi.marshal(
            )[:1000]
            self.ERROR(strError)
            self.setFailure()
            raise RuntimeError(strError)

        crop = sdi.cropBorders
        if len(crop) > 1:
            self.tCrop = tuple([i.value for i in crop])
        elif len(crop) == 1:
            self.tCrop = (crop[0].value, crop[0].value)

        center = sdi.center
        if len(center) > 1:
            self.tCenter = tuple([i.value for i in center])
        elif len(center) == 1:
            self.tCenter = (center[0].value, center[0].value)

        width = sdi.width
        if len(width) > 1:
            self.tWidth = tuple([i.value for i in width])
        elif len(width) == 1:
            self.tWidth = (width[0].value, width[0].value)

        smooth = sdi.smoothBorders
        if len(smooth) == 2:
            self.tSmooth = (smooth[0].value, smooth[1].value)
        elif len(smooth) == 1:
            self.tSmooth = (smooth[0].value, smooth[0].value)

        if sdi.backgroundSubtraction is not None:
            self.bBackgroundsubtraction = (sdi.backgroundSubtraction.value
                                           in [1, True, "true"])

        if sdi.sobelFilter is not None:
            self.sobel = (sdi.sobelFilter in [1, True, "true"])
        EDAssert.equal(self.npaIm1.shape, self.npaIm2.shape,
                       "Images have the same size")

    def process(self, _edObject=None):
        EDPluginExec.process(self)
        self.DEBUG("EDPluginExecMeasureOffsetv1_0.process")
        shape = self.npaIm1.shape
        if (self.tCrop is not None):
            d0min = min(shape[0], self.tCrop[0])
            d1min = min(shape[1], self.tCrop[1])
            d0max = max(0, shape[0] - self.tCrop[0])
            d1max = max(0, shape[1] - self.tCrop[1])
            shape = (d0max - d0min, d1max - d1min)
        else:
            d0min = 0
            d0max = shape[0]
            d1min = 0
            d1max = shape[1]

        if self.tCenter is None:
            #the default center is the geometry center of the image ...
            self.tCenter = [i // 2 for i in shape]
        if self.tWidth is not None:
            d0min = max(0, self.tCenter[0] - (self.tWidth[0] // 2))
            d0max = min(shape[0], d0min + self.tWidth[0])
            d1min = max(0, self.tCenter[1] - (self.tWidth[1] // 2))
            d1max = min(shape[1], d1min + self.tWidth[1])
            shape = (d0max - d0min, d1max - d1min)
        if shape != self.npaIm1.shape:
            self.DEBUG(
                "Redefining ROI to %s - %s ; %s - %s as crop=%s, center=%s and width=%s"
                % (d0min, d0max, d1min, d1max, self.tCrop, self.tCenter,
                   self.tWidth))
            self.npaIm1 = self.npaIm1[d0min:d0max, d1min:d1max]
            self.npaIm2 = self.npaIm2[d0min:d0max, d1min:d1max]
            shape = self.npaIm1.shape
            self.DEBUG("After Crop, images have shape : %s and %s " %
                       (self.npaIm1.shape, self.npaIm2.shape))

        if self.bBackgroundsubtraction:
            self.DEBUG("performing background subtraction")
            x = range(shape[0] - 1) + [shape[0] - 1] * (shape[0] - 1) + range(
                shape[0] - 1, 0, -1) + [0] * (shape[0] - 1)
            y = [0] * (shape[1] - 1) + range(shape[1] - 1) + [shape[1] - 1] * (
                shape[1] - 1) + range(shape[1] - 1, 0, -1)
            spline1 = scipy.interpolate.SmoothBivariateSpline(
                x,
                y, [self.npaIm1[x[i], y[i]] for i in xrange(len(x))],
                kx=1,
                ky=1)
            spline2 = scipy.interpolate.SmoothBivariateSpline(
                x,
                y, [self.npaIm2[x[i], y[i]] for i in xrange(len(x))],
                kx=1,
                ky=1)
            self.npaIm1 -= spline1(range(shape[0]), range(shape[1]))
            self.npaIm2 -= spline2(range(shape[0]), range(shape[1]))

        if self.tSmooth is not None:
            self.npaIm1 *= MeasureOffset.make_mask(shape, self.tSmooth)
            self.npaIm2 *= MeasureOffset.make_mask(shape, self.tSmooth)

        if self.sobel:
            self.npaIm1 = scipy.ndimage.sobel(self.npaIm1)
            self.npaIm2 = scipy.ndimage.sobel(self.npaIm2)

        self.DEBUG("Doing ffts using %s" % self.CONF_CONVOLUTION)
        offset, logs = MeasureOffset.measure_offset(self.npaIm1,
                                                    self.npaIm2,
                                                    self.CONF_CONVOLUTION,
                                                    withLog=True)

        self.tOffset = [XSDataDouble(f) for f in offset]
        logs.insert(0, "")
        if len(logs) <= 4:
            self.DEBUG(os.linesep.join(logs))
        else:
            self.WARNING(os.linesep.join(logs))

    def postProcess(self, _edObject=None):
        EDPluginExec.postProcess(self)
        self.DEBUG("EDPluginExecMeasureOffsetv1_0.postProcess")
        # Create some output data
        self.setDataOutput(XSDataResultMeasureOffset(offset=self.tOffset))
Exemplo n.º 30
0
from __future__ import with_statement

__author__ = "Jérôme Kieffer"
__license__ = "GPLv3+"
__copyright__ = "2011 ESRF"
__status__ = "production"
__date__ = "20120917"

import os, shutil
from EDPluginControl import EDPluginControl
from XSDataCommon import XSDataFile, XSDataString, XSDataStatus
from XSDataEdnaSaxs import XSDataInputAutoSub, XSDataInputDataver, \
    XSDataInputDatcmp, XSDataInputAutoRg, XSDataInputDatop
from XSDataEdnaSaxs import XSDataResultAutoSub
from EDThreading import Semaphore
_sem = Semaphore()


def copy(src, dst):
    """
    Operating system independent copy that does symbolic links on posix system.
    
    This implements a lock to be extremely thread-safe as BsxCube call many times the same plugin ... at the same time 
    """
    with _sem:
        if os.path.exists(src):
            if os.name == "posix":
                if os.path.islink(dst):
                    os.unlink(dst)
                os.symlink(src, dst)
            else:
Exemplo n.º 31
0
class EDPluginExecNormalizeImagev1_1(EDPluginExec):
    """
    Plugin that normalizes an image (subtract dark current and divide by flat-field image, taking into account the exposure time)
    
    v1.1 is compatible with EDShare -> lower memory footprint 
    """
    dictDark = {} #Nota: the key is a string: str(exposutTime)
#    listDarkExposure = []
#    listDarkArray = []
    semaphore = Semaphore()
    dtype = None #
    CONF_DTYPE_KEY = "dtype"
    CONF_DTYPE_DEFAULT = "float32"


    def __init__(self):
        """
        Constructor of the plugin
        """
        EDPluginExec.__init__(self)
        self.setXSDataInputClass(XSDataInputNormalize)
        self.listDataArray = []
        self.listDataExposure = []
        self.listDarkArray = []
        self.listDarkExposure = []
        self.listFlatArray = []
        self.listFlatExposure = []
        self.npaNormalized = None
        self.strOutputFilename = None
        self.strOutputShared = None
        self.shape = None

    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginExecNormalizeImagev1_1.checkParameters")
        self.checkMandatoryParameters(self.getDataInput(), "Data Input is None")


    def configure(self):
        """
        Add the dtype to the configuration file, probably "float32" or "float64"
        """
        EDPluginExec.configure(self)
        self.DEBUG("EDPluginExecNormalizeImagev1_1.configure")
        if self.__class__.dtype is None:
            with self.__class__.semaphore:
                xsPluginItem = self.getConfiguration()
                if (xsPluginItem == None):
                    self.WARNING("EDPluginExecNormalizeImagev1_1.configure: No plugin item defined.")
                    xsPluginItem = XSPluginItem()
                strDtype = EDConfiguration.getStringParamValue(xsPluginItem, self.CONF_DTYPE_KEY)
                if(strDtype == None):
                    self.WARNING("EDPluginExecNormalizeImagev1_1.configure: No configuration parameter found for: %s using default value: %s\n%s"\
                                % (self.CONF_DTYPE_KEY, self.CONF_DTYPE_DEFAULT, xsPluginItem.marshal()))
                    self.__class__.dtype = self.CONF_DTYPE_DEFAULT
                else:
                    self.__class__.dtype = str(strDtype.strip().lower())


    def preProcess(self, _edObject=None):
        EDPluginExec.preProcess(self)
        self.DEBUG("EDPluginExecNormalizeImagev1_1.preProcess")
        sdi = self.getDataInput()
        if sdi.getData() == []:
            strError = "You should either provide at least ONE input filename or an array, you provided: %s" % sdi.marshal()
            self.ERROR(strError)
            self.setFailure()
            raise RuntimeError(strError)
        else:
            for inputData in sdi.getData():
                if inputData.exposureTime is None:
                    self.WARNING("You did not provide an exposure time for DATA... using default: 1")
                    self.listDataExposure.append(1.0)
                else:
                    self.listDataExposure.append(EDUtilsUnit.getSIValue(inputData.exposureTime))
                if inputData.path is not None:
                    strPath = inputData.path.value
                    if os.path.isfile(strPath):
                        self.listDataArray.append(fabio.open(strPath).data)
                    else:
                        strError = "The input file provided for DATA is not a valid file: %s" % strPath
                        self.ERROR(strError)
                        self.setFailure()
                        raise RuntimeError(strError)
                elif inputData.array is not None:
                    self.listDataArray.append(EDUtilsArray.xsDataToArray(inputData.array))
                else:
                    strError = "You should either provide an input filename or an array for DATA, you provided: %s" % inputData.marshal()
                    self.ERROR(strError)
                    self.setFailure()
                    raise RuntimeError(strError)

        for inputFlat in sdi.getFlat():
            if inputFlat.exposureTime is None:
                self.WARNING("You did not provide an exposure time for FLAT... using default: 1")
                expTime = 1.0
            else:
                expTime = EDUtilsUnit.getSIValue(inputFlat.exposureTime)
            self.listFlatExposure.append(expTime)
            if inputFlat.path is not None:
                strPath = inputFlat.path.value
                if os.path.isfile(strPath):
                    self.listFlatArray.append(fabio.open(strPath).data)
                else:
                    strError = "The input file provided for FLAT is not a valid file: %s" % strPath
                    self.ERROR(strError)
                    self.setFailure()
                    raise RuntimeError(strError)
            elif inputFlat.array is not None:
                self.listFlatArray.append(EDUtilsArray.xsDataToArray(inputFlat.array))
            else:
                strError = "You should either provide an input filename or an array for FLAT, you provided: %s" % inputFlat.marshal()
                self.ERROR(strError)
                self.setFailure()
                raise RuntimeError(strError)

        with EDPluginExecNormalizeImagev1_1.semaphore:
            for inputDark in sdi.getDark():
                if inputDark.exposureTime is None:
                    self.WARNING("You did not provide an exposure time for Dark... using default: 1")
                    expTime = 1.0
                else:
                    expTime = EDUtilsUnit.getSIValue(inputDark.exposureTime)
                strMeanDarkKey = "/".join((self.getClassName(), "MeanDark%6.3f" % expTime))
                if strMeanDarkKey not in EDShare:
                    self.listDarkExposure.append(expTime)
                    if inputDark.path is not None:
                        strPath = inputDark.path.value
                        if os.path.isfile(strPath):
                            self.listDarkArray.append(fabio.open(strPath).data)
                        else:
                            strError = "The input file provided for Dark is not a valid file: %s" % strPath
                            self.ERROR(strError)
                            self.setFailure()
                            raise RuntimeError(strError)
                    elif inputDark.array is not None:
                        self.listDarkArray.append(EDUtilsArray.xsDataToArray(inputDark.array))
                    elif inputDark.shared is not None:
                        self.listDarkArray.append(EDShare[inputDark.shared.value])
                    else:
                        strError = "You should either provide an input filename or an array for Dark, you provided: %s" % inputDark.marshal()
                        self.ERROR(strError)
                        self.setFailure()
                        raise RuntimeError(strError)

        if sdi.output is not None:
            if (sdi.output.path is not None):
                self.strOutputFilename = sdi.output.path.value
            elif (sdi.output.shared is not None):
                self.strOutputShared = sdi.output.shared.value
        # else export as array.

        EDAssert.equal(len(self.listDataArray), len(self.listDataExposure), _strComment="number of data images / exposure times ")
        EDAssert.equal(len(self.listFlatArray), len(self.listFlatExposure), _strComment="number of flat images / exposure times ")
        EDAssert.equal(len(self.listDarkArray), len(self.listDarkExposure), _strComment="number of dark images / exposure times ")


    def process(self, _edObject=None):
        EDPluginExec.process(self)
        self.DEBUG("EDPluginExecNormalizeImagev1_1.process")

        #numerator part: 
        fTotalDataTime = 0.0
        self.shape = self.listDataArray[0].shape
        npaSummedData = numpy.zeros(self.shape, dtype=self.dtype)

        for i in range(len(self.listDataArray)):
            fTotalDataTime += self.listDataExposure[i]
            npaSummedData += numpy.maximum(self.listDataArray[i] - self.getMeanDark(self.listDataExposure[i]), 0)
        npaNormalizedData = (npaSummedData / fTotalDataTime).astype(self.dtype)

        #denominator part
        fTotalFlatTime = 0.0
        npaSummedFlat = numpy.zeros(self.shape, dtype=self.dtype)

        for i in range(len(self.listFlatArray)):
            fTotalFlatTime += self.listFlatExposure[i]
            npaSummedFlat += numpy.maximum(self.listFlatArray[i] - self.getMeanDark(self.listFlatExposure[i]), 0)

        npaNormalizedFlat = (npaSummedFlat / fTotalFlatTime).astype(self.dtype)

        self.npaNormalized = npaNormalizedData / numpy.maximum(npaNormalizedFlat, numpy.ones_like(npaNormalizedFlat))
        if self.npaNormalized.dtype != numpy.dtype(self.dtype):
            self.npaNormalized = self.npaNormalized.astype(self.dtype)


    def postProcess(self, _edObject=None):
        EDPluginExec.postProcess(self)
        self.DEBUG("EDPluginExecNormalizeImagev1_1.postProcess")
        xsDataResult = XSDataResultNormalize()
        if self.strOutputFilename is not None:
            self.DEBUG("Writing file %s" % self.strOutputFilename)
            edf = fabio.edfimage.edfimage(data=self.npaNormalized, header={})
            edf.write(self.strOutputFilename)
            xsdo = XSDataImageExt(path=XSDataString(self.strOutputFilename))
        elif self.strOutputShared is not None:
            self.DEBUG("EDShare --> %s" % self.strOutputShared)
            EDShare[ self.strOutputShared] = self.npaNormalized
            xsdo = XSDataImageExt(shared=XSDataString(self.strOutputShared))
        else:
            xsdo = XSDataImageExt(array=EDUtilsArray.arrayToXSData(self.npaNormalized))
        xsDataResult.output = xsdo
        # Create some output data
        self.setDataOutput(xsDataResult)


    def finallyProcess(self, _edObject=None):
        """
        after processing of the plugin:
        remove reference to large objects to save memory
        """
        self.DEBUG("EDPluginExecNormalizeImagev1_1.finallyProcess")
        EDPluginExec.finallyProcess(self, _edObject)
        self.listDataArray = None
        self.listDataExposure = None
        self.listFlatArray = None
        self.listFlatExposure = None
        self.npaNormalized = None


    def getMeanDark(self, _fExposureTime):
        """
        @param _fExposureTime: exposure time 
        @return: mean of darks with this exposure time
        """
        with self.__class__.semaphore:
            if  str(_fExposureTime) not in self.__class__.dictDark:
                npaSumDark = numpy.zeros(self.shape, dtype="float64")
                count = 0
                for fExpTime, npaDark in zip(self.listDarkExposure, self.listDarkArray):
                    if abs(fExpTime - _fExposureTime) / _fExposureTime < 1e-4:
                        npaSumDark += npaDark
                        count += 1
                if count == 0:
                    self.WARNING("No Dark image with Exposure time = %.3f, no dark subtraction" % _fExposureTime)
                    self.__class__.dictDark[str(_fExposureTime) ] = npaSumDark
                else:
                    self.__class__.dictDark[str(_fExposureTime) ] = npaSumDark / float(count)
        return  self.__class__.dictDark[str(_fExposureTime)]
Exemplo n.º 32
0
class EDPluginBioSaxsHPLCv1_2(EDPluginControl):
    """
    plugin for processing Saxs data coming from HPLC

    runs subsequently:
    *ProcessOneFile,
    *subtraction of buffer
    *AutoRg

    todo:
    only store references: Wait for flush to construct HDF5 file and (possibly) web pages with PNG graphs
    
    Changes since v1.0:
    * do not perfom Gnom not Porod analysis
    * return Summed Intensity and time stamp
    * fix issue with timestamp.
    Changes since v1.1:
    * Read configuration for a buffer threshold 
    """

    strControlledPluginProcessOneFile = "EDPluginBioSaxsProcessOneFilev1_4"
    strControlledPluginDatop = "EDPluginExecDatopv2_0"
    strControlledPluginAutoRg = "EDPluginExecAutoRgv1_0"
    strControlledPluginDatCmp = "EDPluginExecDatcmpv2_0"
    strControlledPluginDatAver = "EDPluginExecDataverv2_0"
    dictHPLC = {}  # key=runId, value= HPLCrun instance
    _sem = Semaphore()
    SIMILARITY_THRESHOLD_SAMPLE_KEY = "similaritySample"
    SIMILARITY_THRESHOLD_BUFFER_KEY = "similarityBuffer"
    SIMILARITY_THRESHOLD_SAMPLE_DEFAULT = 0.0
    SIMILARITY_THRESHOLD_BUFFER_DEFAULT = 0.1
    SIMILARITY_THRESHOLD_SAMPLE = None
    SIMILARITY_THRESHOLD_BUFFER = None

    def __init__(self):
        """
        """
        EDPluginControl.__init__(self)
        self.setXSDataInputClass(XSDataInputBioSaxsHPLCv1_0)
        self.edPluginProcessOneFile = None
        self.edPluginSubtract = None
        self.edPluginAutoRg = None
        self.edPluginDatCmp = None
        self.xsDataResult = XSDataResultBioSaxsHPLCv1_0()
        self.runId = None
        self.frameId = None
        self.frame = None
        self.hplc_run = None
        self.curve = None
        self.subtracted = None
        self.lstExecutiveSummary = []
        self.isBuffer = False
        self.scatter_data = None

    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.checkParameters")
        self.checkMandatoryParameters(self.dataInput, "Data Input is None")
        self.checkMandatoryParameters(self.dataInput.rawImage, "No raw image")
        self.checkMandatoryParameters(self.dataInput.sample,
                                      "no Sample parameter")
        self.checkMandatoryParameters(self.dataInput.experimentSetup,
                                      "No experimental setup parameter")

    def configure(self):
        """
        Configures the HPLC plugin by reading from the configuration file 
         - The threshold for similarity with sample & buffer
        """
        EDPluginControl.configure(self)
        if self.__class__.SIMILARITY_THRESHOLD_SAMPLE is None:
            with self._sem:
                if self.__class__.SIMILARITY_THRESHOLD_SAMPLE is None:
                    self.DEBUG("EDPluginBioSaxsHPLCv1_2.configure")
                    self.__class__.SIMILARITY_THRESHOLD_BUFFER = float(
                        self.config.get(
                            self.SIMILARITY_THRESHOLD_BUFFER_KEY,
                            self.SIMILARITY_THRESHOLD_BUFFER_DEFAULT))
                    self.__class__.SIMILARITY_THRESHOLD_SAMPLE = float(
                        self.config.get(
                            self.SIMILARITY_THRESHOLD_SAMPLE_KEY,
                            self.SIMILARITY_THRESHOLD_SAMPLE_DEFAULT))

    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.preProcess")
        sdi = self.dataInput
        if sdi.runId is not None:
            self.runId = sdi.runId.value
        else:
            path = sdi.rawImage.path.value
            if "_" in path:
                self.runId = path[::-1].split("_", 1)[1][::-1]
            else:
                self.runId = path
        with self._sem:
            if self.runId not in self.dictHPLC:
                self.dictHPLC[self.runId] = HPLCrun(self.runId)
        self.hplc_run = self.dictHPLC[self.runId]
        if sdi.frameId is not None:
            self.frameId = sdi.frameId.value
        else:
            path = sdi.rawImage.path.value
            if "_" in path:
                digits = os.path.splitext(
                    os.path.basename(path))[0].split("_")[-1]
                try:
                    self.frameId = int(digits)
                except ValueError:
                    self.WARNING(
                        "frameId is supposed to be an integer, I got %s" %
                        digits)
                    self.frameId = digits
            else:
                self.warning("using frameID=0 in tests, only")
                self.frameId = 0
        with self._sem:
            self.frame = HPLCframe(self.runId, self.frameId)
            self.hplc_run.frames[self.frameId] = self.frame

        if sdi.bufferCurve and os.path.exists(sdi.bufferCurve.path.value):
            with self._sem:
                self.hplc_run.buffer = sdi.bufferCurve.path.value

        if self.hplc_run.hdf5_filename:
            hplc = self.hplc_run.hdf5_filename
        elif sdi.hplcFile:
            hplc = sdi.hplcFile.path.value
        else:
            path = sdi.rawImage.path.value
            dirname = os.path.dirname(os.path.dirname(path))
            hplc_dir = os.path.join(dirname, "HPLC")
            if not os.path.exists(hplc_dir):
                try:
                    os.mkdir(hplc_dir)
                except:
                    pass  # I don't care

            if "_" in path:
                hplc = "_".join(
                    os.path.splitext(
                        os.path.basename(path))[0].split("_")[:-1]) + ".h5"
            else:
                hplc = os.path.splitext(os.path.basename(path))[0] + ".h5"
            hplc = os.path.join(hplc_dir, hplc)
        if not self.hplc_run.hdf5_filename:
            with self._sem:
                self.hplc_run.init_hdf5(hplc)
#        self.xsDataResult.hplcFile = XSDataFile(XSDataString(hplc))

    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.process")

        xsdIn = XSDataInputBioSaxsProcessOneFilev1_0(
            rawImage=self.dataInput.rawImage,
            sample=self.dataInput.sample,
            experimentSetup=self.dataInput.experimentSetup,
            rawImageSize=self.dataInput.rawImageSize,
            normalizedImage=self.dataInput.normalizedImage,
            integratedCurve=self.dataInput.integratedCurve,
            runId=self.dataInput.runId,
            frameId=self.dataInput.frameId)
        self.edPluginProcessOneFile = self.loadPlugin(
            self.strControlledPluginProcessOneFile)
        self.edPluginProcessOneFile.dataInput = xsdIn
        self.edPluginProcessOneFile.connectSUCCESS(
            self.doSuccessProcessOneFile)
        self.edPluginProcessOneFile.connectFAILURE(
            self.doFailureProcessOneFile)
        self.edPluginProcessOneFile.executeSynchronous()

        if self.isFailure():
            return

        xsdIn = XSDataInputDatcmp(inputCurve=[
            XSDataFile(XSDataString(self.hplc_run.first_curve)),
            XSDataFile(XSDataString(self.curve))
        ])
        self.edPluginDatCmp = self.loadPlugin(self.strControlledPluginDatCmp)
        self.edPluginDatCmp.dataInput = xsdIn
        self.edPluginDatCmp.connectSUCCESS(self.doSuccessDatCmp)
        self.edPluginDatCmp.connectFAILURE(self.doFailureDatCmp)
        self.edPluginDatCmp.executeSynchronous()

        if self.isFailure() or self.isBuffer:
            return

        if self.dataInput.subtractedCurve is not None:
            subtracted = self.dataInput.subtractedCurve.path.value
        else:
            subtracted = os.path.splitext(self.curve)[0] + "_sub.dat"
        if self.hplc_run.buffer is not None:
            xsdIn = XSDataInputDatop(inputCurve=[
                XSDataFile(XSDataString(self.curve)),
                XSDataFile(XSDataString(self.hplc_run.buffer))
            ],
                                     outputCurve=XSDataFile(
                                         XSDataString(subtracted)),
                                     operation=XSDataString("sub"))
        else:
            xsdIn = XSDataInputDatop(inputCurve=[
                XSDataFile(XSDataString(self.curve)),
                XSDataFile(XSDataString(self.hplc_run.first_curve))
            ],
                                     outputCurve=XSDataFile(
                                         XSDataString(subtracted)),
                                     operation=XSDataString("sub"))
        self.edPluginDatop = self.loadPlugin(self.strControlledPluginDatop)
        self.edPluginDatop.dataInput = xsdIn
        self.edPluginDatop.connectSUCCESS(self.doSuccessDatop)
        self.edPluginDatop.connectFAILURE(self.doFailureDatop)
        self.edPluginDatop.executeSynchronous()

        if self.subtracted and os.path.exists(self.subtracted):
            self.edPluginAutoRg = self.loadPlugin(
                self.strControlledPluginAutoRg)
            self.edPluginAutoRg.dataInput = XSDataInputAutoRg(
                inputCurve=[XSDataFile(XSDataString(self.subtracted))])
            self.edPluginAutoRg.connectSUCCESS(self.doSuccessAutoRg)
            self.edPluginAutoRg.connectFAILURE(self.doFailureAutoRg)
            self.edPluginAutoRg.executeSynchronous()

    def postProcess(self, _edObject=None):
        """
        after process
        """
        EDPluginControl.postProcess(self)
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.postProcess")
        if self.hplc_run.buffer:
            self.xsDataResult.bufferCurve = XSDataFile(
                XSDataString(self.hplc_run.buffer))
        if self.curve:
            self.xsDataResult.integratedCurve = XSDataFile(
                XSDataString(self.curve))
        if self.subtracted:
            self.xsDataResult.subtractedCurve = XSDataFile(
                XSDataString(self.subtracted))

    def finallyProcess(self, _edObject=None):
        EDPluginControl.finallyProcess(self)
        executiveSummary = os.linesep.join(self.lstExecutiveSummary)
        self.xsDataResult.status = XSDataStatus(
            executiveSummary=XSDataString(executiveSummary))
        self.dataOutput = self.xsDataResult
        if self.frame:
            self.frame.processing = False

    def average_buffers(self):
        """
        Average out all buffers
        """
        self.lstExecutiveSummary.append("Averaging out buffer files: " +
                                        ", ".join(self.hplc_run.for_buffer))
        xsdIn = XSDataInputDataver(inputCurve=[
            XSDataFile(XSDataString(i)) for i in self.hplc_run.for_buffer
        ])
        if self.dataInput.bufferCurve:
            xsdIn.outputCurve = self.dataInput.bufferCurve
        else:
            xsdIn.outputCurve = XSDataFile(
                XSDataString(
                    self.hplc_run.first_curve[::-1].split("_", 1)[1][::-1] +
                    "_buffer_aver%02i.dat" % len(self.hplc_run.for_buffer)))
        self.edPluginDatAver = self.loadPlugin(self.strControlledPluginDatAver)
        self.edPluginDatAver.dataInput = xsdIn
        self.edPluginDatAver.connectSUCCESS(self.doSuccessDatAver)
        self.edPluginDatAver.connectFAILURE(self.doFailureDatAver)
        self.edPluginDatAver.executeSynchronous()

    def doSuccessProcessOneFile(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessProcessOneFile")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessProcessOneFile")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        output = _edPlugin.dataOutput
        if not output.integratedCurve:
            strErr = "Edna plugin ProcessOneFile did not produce integrated curve"
            self.ERROR(strErr)
            self.lstExecutiveSummary.append(strErr)
            self.setFailure()
            return
        self.curve = output.integratedCurve.path.value
        if not os.path.exists(self.curve):
            strErr = "Edna plugin ProcessOneFile: integrated curve not on disk !!"
            self.ERROR(strErr)
            self.lstExecutiveSummary.append(strErr)
            self.setFailure()
            return
        self.xsDataResult.integratedCurve = output.integratedCurve
        self.xsDataResult.normalizedImage = output.normalizedImage
        self.xsDataResult.dataI = output.dataI
        self.xsDataResult.dataQ = output.dataQ
        self.xsDataResult.dataStdErr = output.dataStdErr
        if output.experimentSetup and output.experimentSetup.timeOfFrame:
            startTime = output.experimentSetup.timeOfFrame.value
        else:
            try:
                startTime = float(
                    fabio.openheader(self.dataInput.rawImage.path.value).
                    header["time_of_day"])
            except Exception:
                self.ERROR("Unable to read time_of_day in header of %s" %
                           self.dataInput.rawImage.path.value)
                startTime = 0

        with self._sem:
            if not self.hplc_run.first_curve:
                self.hplc_run.first_curve = self.curve
                self.hplc_run.start_time = startTime
        self.frame.curve = self.curve
        self.frame.time = startTime
        self.xsDataResult.timeStamp = XSDataTime(
            value=(startTime - self.hplc_run.start_time))
        self.calcIntensity()

    def doFailureProcessOneFile(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureProcessOneFile")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureProcessOneFile")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        else:
            self.lstExecutiveSummary.append(
                "Edna plugin ProcessOneFile failed.")
        self.setFailure()

    def doSuccessDatop(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessDatop")
        self.retrieveSuccessMessages(_edPlugin,
                                     "EDPluginBioSaxsHPLCv1_2.doSuccessDatop")
        if _edPlugin and _edPlugin.dataOutput:
            output = _edPlugin.dataOutput
            if output.status and output.status.executiveSummary:
                self.lstExecutiveSummary.append(
                    output.status.executiveSummary.value)
            if output.outputCurve:
                self.subtracted = output.outputCurve.path.value
                if os.path.exists(self.subtracted):
                    self.xsDataResult.subtractedCurve = output.outputCurve
                    self.frame.subtracted = self.subtracted
                else:
                    strErr = "Edna plugin datop did not produce subtracted file %s" % self.subtracted
                    self.ERROR(strErr)
                    self.lstExecutiveSummary.append(strErr)
                    self.setFailure()

    def doFailureDatop(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureDatop")
        self.retrieveFailureMessages(_edPlugin,
                                     "EDPluginBioSaxsHPLCv1_2.doFailureDatop")
        strErr = "Edna plugin datop failed."
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        else:
            self.lstExecutiveSummary.append(strErr)
        self.ERROR(strErr)
        self.setFailure()

    def doSuccessAutoRg(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessAutoRg")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessAutoRg")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        try:
            rg = _edPlugin.dataOutput.autoRgOut[0]
        except:
            self.ERROR("AutoRg returned nothing !")
        else:
            if rg.rg:
                self.frame.Rg = rg.rg.value
            if rg.rgStdev:
                self.frame.Rg_Stdev = rg.rgStdev.value
            if rg.i0:
                self.frame.I0 = rg.i0.value
            if rg.i0Stdev:
                self.frame.I0_Stdev = rg.i0Stdev.value
            if rg.quality:
                self.frame.quality = rg.quality.value
            self.xsDataResult.autoRg = rg
        """
        Calculate the invariants Vc and Qr from the Rambo&Tainer 2013 Paper,
        also the the mass estimate based on Qr for proteins
        """
        if self.subtracted and os.path.exists(self.subtracted):
            self.subtracted_data = numpy.loadtxt(self.subtracted)
            if self.subtracted_data is not None and\
                self.frame.Rg and self.frame.Rg_Stdev and self.frame.I0 and self.frame.I0_Stdev:
                dictRTI = RamboTainerInvariant(self.subtracted_data,
                                               self.frame.Rg,
                                               self.frame.Rg_Stdev,
                                               self.frame.I0,
                                               self.frame.I0_Stdev,
                                               rg.firstPointUsed.value)
                #             {'Vc': vc[0], 'dVc': vc[1], 'Qr': qr, 'dQr': dqr, 'mass': mass, 'dmass': dmass}
                self.frame.Vc = dictRTI.get("Vc")
                self.frame.Vc_Stdev = dictRTI.get("dVc")
                self.frame.Qr = dictRTI.get("Qr")
                self.frame.Qr_Stdev = dictRTI.get("dQ")
                self.frame.mass = dictRTI.get("mass")
                self.frame.mass_Stdev = dictRTI.get("dmass")
                xsdRTI = XSDataRamboTainer(
                    vc=XSDataDouble(self.frame.Vc),
                    qr=XSDataDouble(self.frame.Qr),
                    mass=XSDataDouble(self.frame.mass),
                    dvc=XSDataDouble(self.frame.Vc_Stdev),
                    dqr=XSDataDouble(self.frame.Qr_Stdev),
                    dmass=XSDataDouble(self.frame.mass_Stdev))
                self.xsDataResult.rti = xsdRTI

    def doFailureAutoRg(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureAutoRg")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureAutoRg")
        strErr = "Edna plugin AutoRg failed."
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
            self.lstExecutiveSummary.append(strErr)
        else:
            self.lstExecutiveSummary.append(strErr)
        self.setFailure()

    def doSuccessDatCmp(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessDatCmp")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessDatCmp")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.fidelity:
            fidelity = _edPlugin.dataOutput.fidelity.value
        else:
            strErr = "No fidelity in output of datcmp"
            self.error(strErr)
            self.lstExecutiveSummary.append(strErr)
            #  self.setFailure()
            fidelity = 0
        if self.hplc_run.buffer is None:
            if fidelity > self.SIMILARITY_THRESHOLD_SAMPLE:
                self.isBuffer = True
                if fidelity > self.SIMILARITY_THRESHOLD_BUFFER:
                    self.hplc_run.for_buffer.append(self.curve)
            else:
                self.average_buffers()
        elif fidelity > self.SIMILARITY_THRESHOLD_SAMPLE:
            self.isBuffer = True

    def doFailureDatCmp(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureDatCmp")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureDatCmp")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        else:
            self.lstExecutiveSummary.append("Edna plugin DatCmp failed.")
        self.setFailure()

    def doSuccessDatAver(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessDatAver")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessDatAver")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.outputCurve:
            bufferCurve = _edPlugin.dataOutput.outputCurve.path.value
            if os.path.exists(bufferCurve):
                with self._sem:
                    self.hplc_run.buffer = bufferCurve
            else:
                strErr = "DatAver claimed buffer is in %s but no such file !!!" % bufferCurve
                self.ERROR(strErr)
                self.lstExecutiveSummary.append(strErr)
                self.setFailure()

    def doFailureDatAver(self, _edPlugin=None):
        self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureDatAver")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureDatAver")
        if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary:
            self.lstExecutiveSummary.append(
                _edPlugin.dataOutput.status.executiveSummary.value)
        else:
            self.lstExecutiveSummary.append("Edna plugin DatAver failed.")
        self.setFailure()

    def calcIntensity(self):
        """
        Calculate invarients like:
        Sum(I),
        and set output datastructure.
        
        """
        self.scatter_data = numpy.loadtxt(
            self.xsDataResult.integratedCurve.path.value)
        self.frame.sum_I = self.scatter_data[:, 1].sum()
        self.xsDataResult.summedIntensity = XSDataDouble(
            value=self.frame.sum_I)
Exemplo n.º 33
0
class EDPluginExecAlignFramev1_0(EDPluginExec):
    """
    An exec plugin that taked an images and a reference and alines the frame on the reference based on the SIFT algorithm.
    
    Derived from EDPluginExecMeasureOffsetv2_1.py
    """
    use_sift_pyocl = None
    sift_keypoints = None
    sift_match = None
    keyindex = {
    }  #key = md5 of the image; value = keypoint (as numpy recordarray)
    config_lock = Semaphore()

    def __init__(self):
        """
        """
        EDPluginExec.__init__(self)
        self.setXSDataInputClass(XSDataInputMeasureOffset)
        self.npaIm1 = None
        self.npaIm2 = None
        self.tOffset = None
        self.tCrop = None
        self.tCenter = None
        self.tWidth = None
        self.tSmooth = None
        self.bBackgroundsubtraction = False
        self.sobel = False

    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginExecAlignFramev1_0.checkParameters")
        self.checkMandatoryParameters(self.dataInput, "Data Input is None")

    def configure(self):
        """
        Configures the plugin (if not done yet
        """
        EDPluginExec.configure(self)
        if self.use_sift_pyocl is None:
            with self.config_lock:
                if self.use_sift_pyocl is None:
                    self.DEBUG("EDPluginExecAlignFramev1_0.configure")
                    self.__class__.use_sift_pyocl = self.config.get(
                        "sift_pyocl", False)
                    print self.use_sift_pyocl, self.sift_keypoints, self.sift_match

    def calc_plan(self, image=None):
        if self.sift_keypoints is None:
            with self.config_lock:
                if self.sift_keypoints is None:
                    if self.use_sift_pyocl and sift_pyocl:
                        if self.config.get("device", None):
                            self.__class__.sift_match = sift_pyocl.MatchPlan(
                                device=self.config["device"]).match
                            self.__class__.sift_keypoints = sift_pyocl.SiftPlan(
                                template=image,
                                device=self.config["device"]).keypoints
                        else:
                            deviceType = self.config.get("deviceType", "cpu")
                            self.__class__.sift_match = sift_pyocl.MatchPlan(
                                devicetype=deviceType).match
                            self.__class__.sift_keypoints = sift_pyocl.SiftPlan(
                                template=image,
                                devicetype=deviceType).keypoints
                    elif feature:
                        self.__class__.sift_match = feature.sift_match
                        self.__class__.sift_keypoints = feature.sift_keypoints
                    else:
                        strerr = "Impossible to make plan: use_sift_pyocl=%s but feature=%s and sift_pyocl=%s" % (
                            self.use_sift_pyoc, feature, sift_pyocl)
                        self.ERROR(strerr)
                        raise RuntimeError(strerr)
                    print self.use_sift_pyocl, self.sift_keypoints, self.sift_match

    def preProcess(self, _edObject=None):
        EDPluginExec.preProcess(self)
        self.DEBUG("EDPluginExecAlignFramev1_0.preProcess")
        sdi = self.dataInput
        images = sdi.image
        arrays = sdi.array

        if len(images) == 2:
            self.npaIm1 = numpy.array(EDUtilsArray.getArray(images[0]))
            self.npaIm2 = numpy.array(EDUtilsArray.getArray(images[1]))
        elif len(arrays) == 2:
            self.npaIm1 = EDUtilsArray.xsDataToArray(arrays[0])
            self.npaIm2 = EDUtilsArray.xsDataToArray(arrays[1])
        else:
            strError = "EDPluginExecAlignFramev1_0.preProcess: You should either provide two images or two arrays, but I got: %s" % sdi.marshal(
            )[:1000]
            self.ERROR(strError)
            self.setFailure()
            raise RuntimeError(strError)

        crop = sdi.cropBorders
        if len(crop) > 1:
            self.tCrop = tuple([i.value for i in crop])
        elif len(crop) == 1:
            self.tCrop = (crop[0].value, crop[0].value)

        center = sdi.center
        if len(center) > 1:
            self.tCenter = tuple([i.value for i in center])
        elif len(center) == 1:
            self.tCenter = (center[0].value, center[0].value)

        width = sdi.width
        if len(width) > 1:
            self.tWidth = tuple([i.value for i in width])
        elif len(width) == 1:
            self.tWidth = (width[0].value, width[0].value)

        smooth = sdi.smoothBorders
        if len(smooth) == 2:
            self.tSmooth = (smooth[0].value, smooth[1].value)
        elif len(smooth) == 1:
            self.tSmooth = (smooth[0].value, smooth[0].value)

        if sdi.backgroundSubtraction is not None:
            self.bBackgroundsubtraction = (sdi.backgroundSubtraction.value
                                           in [1, True, "true"])

        if sdi.sobelFilter is not None:
            self.sobel = (sdi.sobelFilter in [1, True, "true"])
        EDAssert.equal(self.npaIm1.shape, self.npaIm2.shape,
                       "Images have the same size")

    def process(self, _edObject=None):
        EDPluginExec.process(self)
        self.DEBUG("EDPluginExecAlignFramev1_0.process")
        shape = self.npaIm1.shape
        if (self.tCrop is not None):
            d0min = min(shape[0], self.tCrop[0])
            d1min = min(shape[1], self.tCrop[1])
            d0max = max(0, shape[0] - self.tCrop[0])
            d1max = max(0, shape[1] - self.tCrop[1])
            shape = (d0max - d0min, d1max - d1min)
        else:
            d0min = 0
            d0max = shape[0]
            d1min = 0
            d1max = shape[1]

        if self.tCenter is None:
            #the default center is the geometry center of the image ...
            self.tCenter = [i // 2 for i in shape]
        if self.tWidth is not None:
            d0min = max(0, self.tCenter[0] - (self.tWidth[0] // 2))
            d0max = min(shape[0], d0min + self.tWidth[0])
            d1min = max(0, self.tCenter[1] - (self.tWidth[1] // 2))
            d1max = min(shape[1], d1min + self.tWidth[1])
            shape = (d0max - d0min, d1max - d1min)
        if shape != self.npaIm1.shape:
            self.DEBUG(
                "Redefining ROI to %s - %s ; %s - %s as crop=%s, center=%s and width=%s"
                % (d0min, d0max, d1min, d1max, self.tCrop, self.tCenter,
                   self.tWidth))
            #array contiguity is needed for checksum calculation
            self.npaIm1 = numpy.ascontiguousarray(self.npaIm1[d0min:d0max,
                                                              d1min:d1max])
            self.npaIm2 = numpy.ascontiguousarray(self.npaIm2[d0min:d0max,
                                                              d1min:d1max])
            shape = self.npaIm1.shape
            self.DEBUG("After Crop, images have shape : %s and %s " %
                       (self.npaIm1.shape, self.npaIm2.shape))

        if self.bBackgroundsubtraction:
            self.DEBUG("performing background subtraction")
            x = range(shape[0] - 1) + [shape[0] - 1] * (shape[0] - 1) + range(
                shape[0] - 1, 0, -1) + [0] * (shape[0] - 1)
            y = [0] * (shape[1] - 1) + range(shape[1] - 1) + [shape[1] - 1] * (
                shape[1] - 1) + range(shape[1] - 1, 0, -1)
            spline1 = scipy.interpolate.SmoothBivariateSpline(
                x,
                y, [self.npaIm1[x[i], y[i]] for i in xrange(len(x))],
                kx=1,
                ky=1)
            spline2 = scipy.interpolate.SmoothBivariateSpline(
                x,
                y, [self.npaIm2[x[i], y[i]] for i in xrange(len(x))],
                kx=1,
                ky=1)
            self.npaIm1 -= spline1(range(shape[0]), range(shape[1]))
            self.npaIm2 -= spline2(range(shape[0]), range(shape[1]))

        if self.tSmooth is not None:
            self.npaIm1 *= MeasureOffset.make_mask(shape, self.tSmooth)
            self.npaIm2 *= MeasureOffset.make_mask(shape, self.tSmooth)

        if self.sobel:
            self.npaIm1 = scipy.ndimage.sobel(self.npaIm1)
            self.npaIm2 = scipy.ndimage.sobel(self.npaIm2)

        self.calc_plan(self.npaIm1)
        checksum1 = hashlib.md5(self.npaIm1).hexdigest()
        checksum2 = hashlib.md5(self.npaIm2).hexdigest()
        if checksum1 not in self.keyindex:
            self.keyindex[checksum1] = self.sift_keypoints(self.npaIm1)
        if checksum2 not in self.keyindex:
            self.keyindex[checksum2] = self.sift_keypoints(self.npaIm2)
        data = self.sift_match(self.keyindex[checksum1],
                               self.keyindex[checksum2])
        v0 = data[:, 0].y - data[:, 1].y
        v1 = data[:, 0].x - data[:, 1].x
        self.tOffset = [
            XSDataDouble(numpy.median(v0)),
            XSDataDouble(numpy.median(v1))
        ]

    def postProcess(self, _edObject=None):
        EDPluginExec.postProcess(self)
        self.DEBUG("EDPluginExecAlignFramev1_0.postProcess")
        # Create some output data
        self.setDataOutput(XSDataResultMeasureOffset(offset=self.tOffset))
Exemplo n.º 34
0
class EDPluginControlAlignStackv1_0(EDPluginControl):
    """
    This control plugin aligns a stack on images in a single HDF5 3D-structure 
    """
    __iRefFrame = None
    __dictRelShift = {
    }  #key=frame number N, value= 2-tuple of shift relative to frame N-1
    __dictAbsShift = {
    }  #key=frame number N, value= 2-tuple of shift relative to frame iRefFrame
    __semaphore = Semaphore()
    MaxOffset = None

    def __init__(self):
        """
        """
        EDPluginControl.__init__(self)
        self.setXSDataInputClass(XSDataInputAlignStack)
        self.iFrames = []
        self.npArrays = []
        self.xsdMeasureOffset = None
        self.hdf5ExtraAttributes = None
        self.xsdHDF5File = None
        self.xsdHDF5Internal = None
        self.bAlwaysMOvsRef = True
        self.bDoAlign = True
        self.semAccumulator = Semaphore()
        self.semMeasure = Semaphore()
        self.semShift = Semaphore()
        self.lstSem = [
            self.locked(), self.semAccumulator, self.semMeasure, self.semShift
        ]
        self.queue = Queue()
        self.__strControlledPluginAccumulator = "EDPluginAccumulatorv1_0"
        self.__strControlledPluginMeasureFFT = "EDPluginExecMeasureOffsetv1_0"
        self.__strControlledPluginMeasureSIFT = "EDPluginExecMeasureOffsetv2_0"
        self.__strControlledPluginShift = "EDPluginExecShiftImagev1_1"
        self.__strControlledPluginHDF5 = "EDPluginHDF5StackImagesv10"

    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        self.DEBUG("EDPluginControlAlignStackv1_0.checkParameters")
        self.checkMandatoryParameters(self.dataInput, "Data Input is None")
        self.checkMandatoryParameters(self.dataInput.getHDF5File(),
                                      "No Output HDF5 file provided")
        self.checkMandatoryParameters(self.dataInput.getInternalHDF5Path(),
                                      "No output HDF5 internal path provided")
        self.checkMandatoryParameters(self.dataInput.images,
                                      "No images to process provided")

    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlAlignStackv1_0.preProcess")

        sdi = self.dataInput
        self.xsdHDF5File = sdi.HDF5File
        self.xsdHDF5Internal = sdi.internalHDF5Path
        self.hdf5ExtraAttributes = sdi.extraAttributes
        if sdi.dontAlign is not None:
            self.bDoAlign = not (bool(sdi.dontAlign.value))

        self.iFrames = [xsd.value for xsd in sdi.index]

        for idx, oneXSDFile in enumerate(sdi.images):
            self.npArrays.append(EDUtilsArray.getArray(oneXSDFile))
            if len(self.iFrames) <= idx:
                if (oneXSDFile.number is not None):
                    self.iFrames.append(oneXSDFile.number.value)
                elif oneXSDFile.path is not None:
                    number = ""
                    filename = oneXSDFile.path.value
                    basename = os.path.splitext(filename)[0]
                    for i in basename[-1:0:-1]:
                        if i.isdigit():
                            number = i + number
                        else:
                            break
                    self.iFrames.append(int(number))

        if self.npArrays == []:
            strError = "EDPluginControlAlignStackv1_0.preProcess: You should either provide an images or an arrays, but I got: %s" % sdi.marshal(
            )
            self.ERROR(strError)
            self.setFailure()

        self.xsdMeasureOffset = sdi.measureOffset
        if self.xsdMeasureOffset.alwaysVersusRef is not None:
            self.bAlwaysMOvsRef = bool(
                self.xsdMeasureOffset.alwaysVersusRef.value)

        with self.__class__.__semaphore:
            if (self.__class__.__iRefFrame is None):
                self.DEBUG("reference Frame is: %s" % sdi.frameReference.value)
                if sdi.getFrameReference() is not None:
                    self.__class__.__iRefFrame = sdi.frameReference.value
                else:
                    self.__class__.__iRefFrame = 0

        if len(self.iFrames) == len(self.npArrays):
            for i, j in zip(self.iFrames, self.npArrays):
                self.addFrame(i, j)
        else:
            self.ERROR(
                "EDPluginControlAlignStackv1_0.preProcess: You should either provide an images with a frame number or precise it in the XML !  I got: %s"
                % sdi.marshal())
            self.setFailure()
            raise RuntimeError

    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlAlignStackv1_0.process")

        for iFrame in self.iFrames:
            edPluginExecAccumulator = self.loadPlugin(
                self.__strControlledPluginAccumulator)
            queryRaw = XSDataQuery()
            queryShift = XSDataQuery()
            queryRaw.setRemoveItems(XSDataBoolean(False))
            queryShift.setRemoveItems(XSDataBoolean(False))
            xsdataAcc = XSDataInputAccumulator()
            if (EDPluginControlAlignStackv1_0.__iRefFrame
                    == iFrame) or (self.bDoAlign == False):
                EDPluginControlAlignStackv1_0.__dictAbsShift[iFrame] = (0.0,
                                                                        0.0)
                EDPluginControlAlignStackv1_0.__dictRelShift[iFrame] = (0.0,
                                                                        0.0)
                self.hdf5_offset(index=iFrame, offset=[0.0, 0.0])
                edPluginExecShift = self.loadPlugin(
                    self.__strControlledPluginShift)
                xsdata = XSDataInputShiftImage(
                    index=XSDataInteger(iFrame),
                    offset=[
                        XSDataDouble(i) for i in
                        EDPluginControlAlignStackv1_0.__dictAbsShift[iFrame]
                    ],
                    inputImage=self.getFrameRef(iFrame),
                    outputImage=XSDataImageExt(
                        shared=XSDataString("Shifted-%06i" % iFrame)))
                edPluginExecShift.setDataInput(xsdata)
                edPluginExecShift.connectSUCCESS(self.doSuccessExecShiftImage)
                edPluginExecShift.connectFAILURE(self.doFailureExecShiftImage)
                self.queue.put(edPluginExecShift)
                if (self.bDoAlign == False):
                    self.executeControlledPlugins()
                    return

            elif EDPluginControlAlignStackv1_0.__iRefFrame < iFrame:
                if self.bAlwaysMOvsRef:
                    queryRaw.setItem([
                        XSDataString(
                            "raw %04i" %
                            (EDPluginControlAlignStackv1_0.__iRefFrame)),
                        XSDataString("raw %04i" % iFrame)
                    ])
                    xsdataAcc.setQuery([queryRaw])
                else:
                    queryRaw.setItem([
                        XSDataString("raw %04i" % (iFrame - 1)),
                        XSDataString("raw %04i" % iFrame)
                    ])
                    queryShift.setItem([
                        XSDataString("shift %04i" % i) for i in range(
                            EDPluginControlAlignStackv1_0.__iRefFrame +
                            1, iFrame + 1)
                    ])
                    xsdataAcc.setQuery([queryRaw, queryShift])
            elif EDPluginControlAlignStackv1_0.__iRefFrame > iFrame:
                if self.bAlwaysMOvsRef:
                    queryRaw.setItem([
                        XSDataString("raw %04i" % iFrame),
                        XSDataString(
                            "raw %04i" %
                            (EDPluginControlAlignStackv1_0.__iRefFrame))
                    ])
                    xsdataAcc.setQuery([queryRaw])
                else:
                    queryRaw.setItem([
                        XSDataString("raw %04i" % (iFrame + 1)),
                        XSDataString("raw %04i" % iFrame)
                    ])
                    queryShift.setItem([
                        XSDataString("shift %04i" % i) for i in range(
                            EDPluginControlAlignStackv1_0.__iRefFrame -
                            1, iFrame - 1, -1)
                    ])
                    xsdataAcc.setQuery([queryRaw, queryShift])
            if (EDPluginControlAlignStackv1_0.__iRefFrame == iFrame):
                self.saveReferenceFrame(iFrame)

            xsdataAcc.setItem([XSDataString("raw %04i" % iFrame)])
            edPluginExecAccumulator.setDataInput(xsdataAcc)
            edPluginExecAccumulator.connectSUCCESS(
                self.doSuccessExecAccumultor)
            edPluginExecAccumulator.connectFAILURE(
                self.doFailureExecAccumulator)
            self.queue.put(edPluginExecAccumulator)
        self.executeControlledPlugins()

    def executeControlledPlugins(self):
        """
        Execute all plugins under control: 
        """
        bAllFinished = False
        while not bAllFinished:
            #acquire all semaphores to be sure no plugins are under configuration !
            for sem in self.lstSem:
                with sem:
                    pass
            if self.queue.empty():
                self.synchronizePlugins()
                bAllFinished = self.queue.empty()
            else:
                while not self.queue.empty():
                    try:
                        plugin = self.queue.get()
                    except Exception:
                        self.WARNING(
                            "In EDPluginControlAlignStackv1_0, exception in self.queue.get()"
                        )
                        break
                    else:
                        #this is a hack to prevent thousands of threads to be launched at once.
                        with EDUtilsParallel.getSemaphoreNbThreads():
                            pass
                        plugin.execute()
                self.synchronizePlugins()

    def postProcess(self, _edObject=None):
        EDPluginControl.postProcess(self)
        self.DEBUG("EDPluginControlAlignStackv1_0.postProcess")
        self.executeControlledPlugins()
        self.synchronizePlugins()
        # Create some output data
        xsDataResult = XSDataResultAlignStack()
        xsDataResult.setHDF5File(self.xsdHDF5File)
        xsDataResult.setInternalHDF5Path(self.xsdHDF5Internal)
        self.setDataOutput(xsDataResult)
        self.emptyListOfLoadedPlugin()

    def doSuccessExecMeasureOffset(self, _edPlugin=None):
        with self.semMeasure:
            self.DEBUG(
                "EDPluginControlAlignStackv1_0.doSuccessExecMeasureOffset")
            self.retrieveSuccessMessages(
                _edPlugin,
                "EDPluginControlAlignStackv1_0.doSuccessExecMeasureOffset")
            listIndex = [i.getValue() for i in _edPlugin.dataInput.index]
            listIndex.sort()
            dataOutput = _edPlugin.dataOutput
            if self.bAlwaysMOvsRef:
                if min(listIndex) < EDPluginControlAlignStackv1_0.__iRefFrame:
                    iToShift, iRef = tuple(listIndex)
                    EDPluginControlAlignStackv1_0.__dictAbsShift[
                        iToShift] = tuple(
                            [-i.getValue() for i in dataOutput.getOffset()])
                else:
                    iRef, iToShift = tuple(listIndex)
                    EDPluginControlAlignStackv1_0.__dictAbsShift[
                        iToShift] = tuple(
                            [i.getValue() for i in dataOutput.getOffset()])
                self.screen(
                    "Frame number %i has absolute offset of %.3f,%.3f" %
                    (iToShift,
                     EDPluginControlAlignStackv1_0.__dictAbsShift[iToShift][0],
                     EDPluginControlAlignStackv1_0.__dictAbsShift[iToShift][1])
                )
                edPluginExecShift = self.loadPlugin(
                    self.__strControlledPluginShift)
                xsdata = XSDataInputShiftImage(
                    index=XSDataInteger(iToShift),
                    offset=[
                        XSDataDouble(i) for i in
                        EDPluginControlAlignStackv1_0.__dictAbsShift[iToShift]
                    ],
                    inputImage=self.getFrameRef(iToShift),
                    outputImage=XSDataImageExt(
                        shared=XSDataString("Shifted-%06i" % iToShift)))
                edPluginExecShift.setDataInput(xsdata)
                edPluginExecShift.connectSUCCESS(self.doSuccessExecShiftImage)
                edPluginExecShift.connectFAILURE(self.doFailureExecShiftImage)
                self.queue.put(edPluginExecShift)
            else:
                if min(listIndex) < EDPluginControlAlignStackv1_0.__iRefFrame:

                    iToShift, iRef = tuple(listIndex)
                    EDPluginControlAlignStackv1_0.__dictRelShift[
                        iToShift] = tuple(
                            [-i.value for i in dataOutput.offset])
                else:
                    iRef, iToShift = tuple(listIndex)
                    EDPluginControlAlignStackv1_0.__dictRelShift[
                        iToShift] = tuple([i.value for i in dataOutput.offset])
                self.screen(
                    "Frame number %i has relative offset of %.3f,%.3f" %
                    (iToShift,
                     EDPluginControlAlignStackv1_0.__dictRelShift[iToShift][0],
                     EDPluginControlAlignStackv1_0.__dictRelShift[iToShift][1])
                )

                xsdata = XSDataInputAccumulator(
                    item=[XSDataString("shift %04i" % iToShift)])
                edPluginExecAccumulator = self.loadPlugin(
                    self.__strControlledPluginAccumulator)
                edPluginExecAccumulator.setDataInput(xsdata)
                edPluginExecAccumulator.connectSUCCESS(
                    self.doSuccessExecAccumultor)
                edPluginExecAccumulator.connectFAILURE(
                    self.doFailureExecAccumulator)
                self.queue.put(edPluginExecAccumulator)

    def doFailureExecMeasureOffset(self, _edPlugin=None):
        self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecMeasureOffset")
        self.retrieveFailureMessages(
            _edPlugin,
            "EDPluginControlAlignStackv1_0.doFailureExecMeasureOffset")
        self.ERROR(
            "Failure in execution of the MeasureOffset with input: %s and output %s"
            % (_edPlugin.dataInput.marshal()[:1000],
               _edPlugin.dataOutput.marshal()[:1000]))
        self.setFailure()

    def doSuccessExecShiftImage(self, _edPlugin=None):
        with self.semShift:
            edPluginExecHDF5 = self.loadPlugin(self.__strControlledPluginHDF5)
            self.DEBUG("EDPluginControlAlignStackv1_0.doSuccessExecShiftImage")
            self.retrieveSuccessMessages(
                _edPlugin,
                "EDPluginControlAlignStackv1_0.doSuccessExecShiftImage")
            xsdIdx = _edPlugin.dataInput.index
            self.__class__.MaxOffset = _edPlugin.MAX_OFFSET_VALUE
            self.hdf5_offset(
                index=xsdIdx.value,
                offset=[i.value for i in _edPlugin.dataInput.offset])
            xsdata = XSDataInputHDF5StackImages(
                chunkSegmentation=XSDataInteger(32),
                forceDtype=XSDataString("float32"),
                extraAttributes=self.hdf5ExtraAttributes,
                internalHDF5Path=self.xsdHDF5Internal,
                HDF5File=self.xsdHDF5File,
                index=[xsdIdx],
                inputImageFile=[_edPlugin.dataOutput.outputImage])
            #                                                inputArray=[_edPlugin.dataOutput.outputArray])
            edPluginExecHDF5.setDataInput(xsdata)
            edPluginExecHDF5.connectSUCCESS(self.doSuccessExecStackHDF5)
            edPluginExecHDF5.connectFAILURE(self.doFailureExecStackHDF5)
            self.queue.put(edPluginExecHDF5)

    def doFailureExecShiftImage(self, _edPlugin=None):
        self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecShiftImage")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginControlAlignStackv1_0.doFailureExecShiftImage")
        self.ERROR(
            "Failure in execution of the ExecShiftImage with input: %s and output %s"
            % (_edPlugin.dataInput.marshal()[:1000],
               _edPlugin.dataOutput.marshal()[:1000]))
        self.setFailure()

    def doSuccessExecStackHDF5(self, _edPlugin=None):
        self.DEBUG("EDPluginControlAlignStackv1_0.doSuccessExecStackHDF5")
        self.retrieveSuccessMessages(
            _edPlugin, "EDPluginControlAlignStackv1_0.doSuccessExecStackHDF5")

    def doFailureExecStackHDF5(self, _edPlugin=None):
        self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecStackHDF5")
        self.retrieveFailureMessages(
            _edPlugin, "EDPluginControlAlignStackv1_0.doFailureExecStackHDF5")
        self.ERROR(
            "Failure in execution of the ExecStackHDF5 with input: %s " %
            (_edPlugin.dataInput.marshal()[:1000]))
        if _edPlugin.dataOutput is not None:
            self.ERROR(
                "Failure in execution of the ExecStackHDF5 with output %s" %
                (_edPlugin.dataOutput.marshal()[:1000]))

    def doSuccessExecAccumultor(self, _edPlugin=None):
        with self.semAccumulator:
            self.DEBUG("EDPluginControlAlignStackv1_0.doSuccessExecAccumultor")
            self.retrieveSuccessMessages(
                _edPlugin,
                "EDPluginControlAlignStackv1_0.doSuccessExecAccumultor")
            for query in _edPlugin.dataOutput.getQuery():
                self.addExtraTime(60)
                _edPlugin.addExtraTime(60)
                accType = query.getItem()[0].getValue().split()[0]
                listInt = [
                    int(i.getValue().split()[1]) for i in query.getItem()
                ]
                if accType == "raw":
                    listFrame = [self.getFrameRef(i) for i in listInt]

                    xsdata = XSDataInputMeasureOffset(image=listFrame)
                    doSIFT = False
                    if self.xsdMeasureOffset is not None:
                        xsdata.setCropBorders(
                            self.xsdMeasureOffset.getCropBorders())
                        xsdata.setSmoothBorders(
                            self.xsdMeasureOffset.getSmoothBorders())
                        xsdata.setBackgroundSubtraction(
                            self.xsdMeasureOffset.getRemoveBackground())
                        if self.xsdMeasureOffset.useSift is not None:
                            doSIFT = self.xsdMeasureOffset.useSift.value
                    if max(listInt
                           ) > EDPluginControlAlignStackv1_0.__iRefFrame:
                        listInt.sort()
                    else:
                        listInt.sort(reverse=True)
                    xsdata.setIndex([XSDataInteger(i) for i in listInt])
                    if doSIFT:
                        edPluginExecMeasure = self.loadPlugin(
                            self.__strControlledPluginMeasureSIFT)
                    else:
                        edPluginExecMeasure = self.loadPlugin(
                            self.__strControlledPluginMeasureFFT)
                    edPluginExecMeasure.setDataInput(xsdata)
                    edPluginExecMeasure.connectSUCCESS(
                        self.doSuccessExecMeasureOffset)
                    edPluginExecMeasure.connectFAILURE(
                        self.doFailureExecMeasureOffset)
                    self.queue.put(edPluginExecMeasure)

                elif accType == "shift":
                    shift_1 = 0.0
                    shift_2 = 0.0

                    for frame in listInt:
                        shift_1 += EDPluginControlAlignStackv1_0.__dictRelShift[
                            frame][0]
                        shift_2 += EDPluginControlAlignStackv1_0.__dictRelShift[
                            frame][1]
                    if listInt[0] > EDPluginControlAlignStackv1_0.__iRefFrame:
                        iFrameShift = max(listInt)
                    else:
                        iFrameShift = min(listInt)
                    EDPluginControlAlignStackv1_0.__dictAbsShift[
                        iFrameShift] = (shift_1, shift_2)
                    self.screen(
                        "Frame number %i has absolute offset of %.3f,%.3f" %
                        (iFrameShift, shift_1, shift_2))

                    edPluginExecShift = self.loadPlugin(
                        self.__strControlledPluginShift)
                    edPluginExecShift.dataInput = XSDataInputShiftImage(
                        index=XSDataInteger(iFrameShift),
                        offset=[XSDataDouble(shift_1),
                                XSDataDouble(shift_2)],
                        inputImage=self.getFrameRef(iFrameShift),
                        outputImage=XSDataImageExt(
                            shared=XSDataString("Shifted-%06i" % iFrameShift)))

                    edPluginExecShift.connectSUCCESS(
                        self.doSuccessExecShiftImage)
                    edPluginExecShift.connectFAILURE(
                        self.doFailureExecShiftImage)
                    self.queue.put(edPluginExecShift)
            self.DEBUG("Items: %s" % EDPluginAccumulatorv1_0.getItems())
            self.DEBUG("Queries: %s" % EDPluginAccumulatorv1_0.getQueries())

    def doFailureExecAccumulator(self, _edPlugin=None):
        self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecAccumulator")
        self.retrieveFailureMessages(
            _edPlugin,
            "EDPluginControlAlignStackv1_0.doFailureExecAccumulator")
        self.ERROR(
            "Failure in execution of the accumulator with input: %s and output %s"
            % (_edPlugin.dataInput.marshal()[:1000],
               _edPlugin.dataOutput.marshal()[:1000]))
        self.setFailure()

    def saveReferenceFrame(self, iFrame):
        """
        Save the reference frame to the HDF5 file
        @param iFrame: frame number to save as reference frame
        """
        ref = "reference_frame"
        hdf5file = self.xsdHDF5File.path.value
        edpluginHDF5 = self.loadPlugin(self.__strControlledPluginHDF5)
        with edpluginHDF5.getFileLock(hdf5file):
            #Seems strange to redefine h5Grp but if there is a flush in between: h5Grp could be closed
            entry = edpluginHDF5.getHDF5File(hdf5file)[
                self.xsdHDF5Internal.value]
            if ref in entry:
                del entry[ref]
            entry[ref] = self.getFrame(iFrame)
            entry[ref].attrs["index"] = iFrame

    def hdf5_offset(self, index, offset):
        with EDPluginHDF5.getFileLock(self.xsdHDF5File.path.value):
            grp = EDPluginHDF5.getHDF5File(
                self.xsdHDF5File.path.value)[self.xsdHDF5Internal.value]
            ds = grp["Offsets"]
            if self.MaxOffset:
                if "MaxOffset" not in ds.attrs:
                    ds.attrs["MaxOffset"] = self.MaxOffset
            ds[index, :] = offset

    @classmethod
    def showData(cls):
        EDVerbose.screen("*" * 20 + "EDPluginControlAlignStackv1_0" + "*" * 20)
        EDVerbose.screen("Reference Frame: %s" % cls.__iRefFrame)
        if len(cls.__dictRelShift) < len(cls.__dictAbsShift):
            mydict = cls.__dictAbsShift.copy()
        else:
            mydict = cls.__dictRelShift.copy()
        lstTxt = []
        for i in mydict:
            txt = "Frame %4i:\t" % i
            rela = cls.__dictRelShift.get(i)
            abso = cls.__dictAbsShift.get(i)
            if rela:
                txt += "relative: (%.3f, %.3f)\t" % rela
            else:
                txt += "relative: %12s\t" % rela
            if abso:
                txt += "absolute:  (%.3f, %.3f)" % abso
            else:
                txt += "absolute:  %12s" % abso
            lstTxt.append(txt)
        EDVerbose.screen(os.linesep.join(lstTxt))
        items = EDPluginAccumulatorv1_0.getItems()
        items.sort()
        EDVerbose.screen("Items in the accumulator: %s" % (items))
        querylist = [" "] + [
            str(i) for i in EDPluginAccumulatorv1_0.getQueries().keys()
        ]
        EDVerbose.screen("Queries in the accumulator: " +
                         os.linesep.join(querylist))

    @classmethod
    def addFrame(cls, index, value):
        """
        Just store the value to EDShare 
        """
        key = "Normalized-%06i" % int(index)
        if key not in EDShare:
            EDShare[key] = value

    @classmethod
    def getFrame(cls, index):
        """
        Just Retrieves the value from EDShare 
        """
        return EDShare["Normalized-%06i" % int(index)]

    @classmethod
    def getFrameRef(cls, index):
        """
        Just retrieves the reference in the EDShare store 
        @return: reference to the frame in EDShare
        @rtype: XSDataImageExt
        """
        return XSDataImageExt(shared=XSDataString("Normalized-%06i" %
                                                  int(index)))

    @classmethod
    def cleanUp(cls):
        """
        Clean up of the dictionary containing images: Left for compatibility reasons
        """
        EDShare.flush()
Exemplo n.º 35
0
class EDUtilsParallel(object):
    """ 
    A class helping to control the workload on a multi-threaded application. 
    """
    _iActualNbCPU = None
    _semaphore = Semaphore()
    _semaphoreInit = Semaphore()
    _semaphoreNbThreads = None
    _iNbThreads = None

    @classmethod
    def _detectNumberOfCPUs(cls):
        """
        detect the actual number of CPUs and stores it in a class variable 
        """
        with cls._semaphore:
            if cls._iActualNbCPU is not None:
                return
            iNbCPU = None
            #The best way: using python 2.6 or later
            try:
                import multiprocessing
                iNbCPU = multiprocessing.cpu_count()

            except (ImportError, NotImplementedError):
                iNbCPU = None

            # POSIX
            if iNbCPU is None:
                try:
                    iNbCPU = int(os.sysconf('SC_NPROCESSORS_ONLN'))
                    if iNbCPU <= 0:
                        iNbCPU = None
                except (AttributeError, ValueError):
                    iNbCPU = None

            # Windows
            if iNbCPU is None:
                try:
                    iNbCPU = int(os.environ['NUMBER_OF_PROCESSORS'])
                    if iNbCPU <= 0:
                        iNbCPU = None
                except (KeyError, ValueError):
                    iNbCPU = None

            # jython
            if iNbCPU is None:
                try:
                    from java.lang import Runtime
                    runtime = Runtime.getRuntime()
                    iNbCPU = runtime.availableProcessors()
                    if iNbCPU <= 0:
                        iNbCPU = None
                except ImportError:
                    iNbCPU = None

            # BSD
            if iNbCPU is None:
                try:
                    sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'],
                                              stdout=subprocess.PIPE)
                    scStdout = sysctl.communicate()[0]
                    iNbCPU = int(scStdout)
                    if iNbCPU <= 0:
                        iNbCPU = None
                except (OSError, ValueError):
                    iNbCPU = None

            # Linux
            if iNbCPU is None:
                try:
                    iNbCPU = open('/proc/cpuinfo').read().count('processor\t:')
                    if iNbCPU <= 0:
                        iNbCPU = None
                except IOError:
                    iNbCPU = None

            # Solaris
            if iNbCPU is None:
                try:
                    pseudoDevices = os.listdir('/devices/pseudo/')
                    expr = re.compile('^cpuid@[0-9]+$')
                    iNbCPU = 0
                    for pd in pseudoDevices:
                        if expr.match(pd) != None:
                            iNbCPU += 1
                    if iNbCPU <= 0:
                        iNbCPU = None
                except OSError:
                    iNbCPU = None

            # Other UNIXes (heuristic)
            if iNbCPU is None:
                try:
                    try:
                        dmesg = open('/var/run/dmesg.boot').read()
                    except IOError:
                        dmesgProcess = subprocess.Popen(['dmesg'],
                                                        stdout=subprocess.PIPE)
                        dmesg = dmesgProcess.communicate()[0]
                    iNbCPU = 0
                    while '\ncpu' + str(iNbCPU) + ':' in dmesg:
                        iNbCPU += 1
                    if iNbCPU <= 0:
                        iNbCPU = None
                except OSError:
                    iNbCPU = None

            #if nothing else works ...
            if iNbCPU is None:
                iNbCPU = 1

            cls._iActualNbCPU = iNbCPU

    @classmethod
    def detectNumberOfCPUs(cls, _iMaxCPU=sys.maxint, _bForce=False):
        """
        class method :
        Detects the number of CPUs on a system. Cribbed from pp.
        found : http://www.boduch.ca/2009/06/python-cpus.html
        
        @param _iMaxCPU: the maximum number of CPUs allowed 
        @type _iMaxCPU: integer
        @param  _bForce: force the number of CPUs for debugging
        @type _bForce: boolean
        @return: the number of CPUs
        @rtype: integer
        """
        if not isinstance(_iMaxCPU, int):
            _iMaxCPU = sys.maxint
        else:
            _iMaxCPU = max(1, _iMaxCPU)

        if isinstance(_bForce, bool):
            if _bForce:
                return _iMaxCPU

        if cls._iActualNbCPU is None:
            cls._detectNumberOfCPUs()
        iNbCPU = cls._iActualNbCPU
        if iNbCPU < 1:
            return 1
        elif iNbCPU > _iMaxCPU:
            return _iMaxCPU
        else:
            return iNbCPU

    @classmethod
    def initializeNbThread(cls, _iNbThread=None):
        """
        Class method:
        Initialises a semaphore with the right number of threads
        @param _iNbThread: the maximum number of threads concurrently running and CPU intensive
        @type _iNbThread: integer
        """
        with cls._semaphoreInit:
            if cls._semaphoreNbThreads is None:
                if _iNbThread is None:
                    _iNbThread = cls.detectNumberOfCPUs()
                EDVerbose.DEBUG(
                    "Initializing EDUtilsParallel semaphoreNbThreads to %s" %
                    _iNbThread)
                cls._iNbThreads = _iNbThread
                cls._semaphoreNbThreads = Semaphore(_iNbThread)
            else:
                if cls._iNbThreads != _iNbThread:
                    EDVerbose.WARNING(
                        "cls._semaphoreNbThreads was already initialized to %s, not changing to %s"
                        % (cls._iNbThreads, _iNbThread))

    @classmethod
    def uninitializeNbThread(cls):
        """
        For testing purpose: un-initialize the semaphore controlling the number of execPlugin running at once  
        """
        if cls._semaphoreNbThreads is not None:
            EDVerbose.DEBUG(
                "resetting the semaphore concerning the number of threads.")
            cls._semaphoreNbThreads = None

    @classmethod
    def getSemaphoreValue(cls):
        """
        Class method:
        getter for the current value of the semaphore counting the CPU-active threads
        
        @return: value of the semaphore (or None if not initialized) 
        @rtype: int
        """
        if cls._semaphoreNbThreads is not None:
            return cls._semaphoreNbThreads._Semaphore__value
        else:
            return None

    @classmethod
    def getNbRunning(cls):
        """
        Class method:
        getter for the number of CPU-active threads running
    
        @return: the number of CPU-active threads runnings
        @rtype: integer
        """
        if cls._semaphoreNbThreads is not None:
            return cls._iNbThreads - cls._semaphoreNbThreads._Semaphore__value
        else:
            return None

    @classmethod
    def semaphoreNbThreadsAcquire(cls):
        """
        Class method:
        Method to acquire the semaphore that controls the number of threads running concurrently
        """
        if cls._semaphoreNbThreads is not None:
            cls._semaphoreNbThreads.acquire()
        else:
            EDVerbose.DEBUG(
                "EDUtilsParallel: Unable to acquire an uninitialized semaphore (NbCPU)."
            )

    @classmethod
    def semaphoreNbThreadsRelease(cls):
        """
        Class method:
        Method to release the semaphore that controls the number of threads running concurrently
        """
        if cls._semaphoreNbThreads is not None:
            if cls._semaphoreNbThreads._Semaphore__value < cls._iNbThreads:
                cls._semaphoreNbThreads.release()
        else:
            EDVerbose.DEBUG(
                "EDUtilsParallel: Unable to release an uninitialized semaphore (NbCPU)."
            )

    @classmethod
    def getSemaphoreNbThreads(cls):
        """
        classmethod that returns the semaphore nb threads (to be used with the with_statement)
        @return: semaphore controling CPUs
        """
        if cls._semaphoreNbThreads is not None:
            return cls._semaphoreNbThreads
        else:
            EDVerbose.DEBUG(
                "EDUtilsParallel: Unable to acquire an uninitialized semaphore (NbCPU)."
            )