def __init__(self): """ """ EDPluginControl.__init__(self) self.setXSDataInputClass(XSDataInputAlignStack) self.iFrames = [] self.npArrays = [] self.xsdMeasureOffset = None self.hdf5ExtraAttributes = None self.xsdHDF5File = None self.xsdHDF5Internal = None self.bAlwaysMOvsRef = True self.bDoAlign = True self.semAccumulator = Semaphore() self.semMeasure = Semaphore() self.semShift = Semaphore() self.lstSem = [ self.locked(), self.semAccumulator, self.semMeasure, self.semShift ] self.queue = Queue() self.__strControlledPluginAccumulator = "EDPluginAccumulatorv1_0" self.__strControlledPluginMeasureFFT = "EDPluginExecMeasureOffsetv1_0" self.__strControlledPluginMeasureSIFT = "EDPluginExecMeasureOffsetv2_0" self.__strControlledPluginShift = "EDPluginExecShiftImagev1_1" self.__strControlledPluginHDF5 = "EDPluginHDF5StackImagesv10"
def __init__(self, _strPluginName, _functXMLin, \ _functXMLout=None, _functXMLerr=None, \ _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None): """ This is the constructor of the edna plugin launcher. @param _strPluginName: the name of the ENDA plugin @type _strPluginName: python string @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. @type _functXMLin: python function @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout @type _functXMLOut: python function @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin @type _functXMLErr: python function @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores will be auto-detected. @type _iNbThreads: python integer @param _fDelay: The delay in seconds between two directories analysis @type _fDelay: python float @param _bVerbose: Do you want the EDNA plugin execution to be verbose ? @type _bVerbose: boolean @param _bDebug: Do you want EDNA plugin execution debug output (OBS! very verbose) ? @type _bDebug: boolean """ EDLogging.__init__(self) self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads) EDUtilsParallel.initializeNbThread(self.__iNbThreads) ################################################################################ # #We are not using the one from EDUtilsParallel to leave it to control the number of execPlugins. ################################################################################ self.__semaphoreNbThreads = Semaphore(self.__iNbThreads) self.__strPluginName = _strPluginName self.__functXMLin = _functXMLin self.__functXMLout = _functXMLout self.__functXMLerr = _functXMLerr self.__strCurrWorkDir = os.getcwd() self.__strTempDir = None self.__listInputPaths = [] self.__dictCurrentlyRunning = {} if _bVerbose is not None: if _bVerbose: self.setVerboseDebugOn() else: self.setVerboseOff() if _bDebug is not None: if _bDebug: self.setVerboseDebugOn() else: self.setVerboseDebugOff() self.__fDelay = _fDelay #default delay between two directory checks. self.__bQuit = False # To check if we should quit the application self.__bIsFirstExecute = True signal.signal(signal.SIGTERM, self.handleKill) signal.signal(signal.SIGINT, self.handleKill)
def __init__(self): object.__init__(self) self.__pySemaphoreWrite = Semaphore() self.__bIsVerbose = True self.__bIsVerboseDebug = False self.__bIsTest = False self.__edLogFile = EDLogFile()
def __init__(self, runId, first_curve=None): self.id = runId self.buffer = None self.first_curve = first_curve self.frames = {} #key: id, value: HPLCframe instance self.curves = [] self.for_buffer = [] self.hdf5_filename = None self.hdf5 = None self.chunk_size = 250 self.lock = Semaphore() if first_curve: self.files.append(first_curve) self.max_size = None self.time = None self.gnom = None self.Dmax = None self.total = None self.volume = None self.Rg = None self.Rg_Stdev = None self.I0 = None self.I0_Stdev = None self.quality = None self.q = None self.size = None
def parse(self): """ parse options from command line """ parser = optparse.OptionParser() parser.add_option("-V", "--version", dest="version", action="store_true", help="print version of the program and quit", metavar="FILE", default=False) parser.add_option("-v", "--verbose", action="store_true", dest="debug", default=False, help="switch to debug/verbose mode") parser.add_option("-m", "--mask", dest="mask", help="file containing the mask (for image reconstruction)", default=None) parser.add_option("-M", "--mode", dest="mode", help="Mode can be online/offline/all", default="offline") parser.add_option("-o", "--out", dest="output", help="file for log", default=None) parser.add_option("-w", "--wavelength", dest="wavelength", type="float", help="wavelength of the X-Ray beam in Angstrom", default=None) parser.add_option("-e", "--energy", dest="energy", type="float", help="energy of the X-Ray beam in keV (hc=%skeV.A)" % self.hc, default=None) parser.add_option("-t", "--template", dest="template", type="str", help="template XML file", default=None) parser.add_option("-n", "--nbcpu", dest="nbcpu", type="int", help="template XML file", default=self.nbcpu) (options, args) = parser.parse_args() # Analyse aruments and options if options.version: print("BioSaxs Azimuthal integration version %s" % __version__) sys.exit(0) if options.debug: EDVerbose.setVerboseDebugOn() self.debug = True if options.output: EDVerbose.setLogFileName(options.output) if options.mask and os.path.isfile(options.mask): self.maskfile = options.mask if options.template and os.path.isfile(options.template): self.xml = open(options.template).read() if options.wavelength: self.wavelength = 1e-10 * options.wavelength elif options.energy: self.wavelength = 1e-10 * self.hc / options.energy if options.mode=="offline": self.mode = "offline" self.newerOnly = False elif options.mode=="online": self.mode = "dirwarch" self.newerOnly = True elif options.mode=="dirwatch": self.mode = "dirwarch" self.newerOnly = False self.cpu_sem = Semaphore(options.nbcpu) self.nbcpu = options.nbcpu self.dataFiles = [f for f in args if os.path.isfile(f)] if not self.dataFiles: raise RuntimeError("Please provide datafiles or read the --help")
def __init__(self, strPluginName, iNbCpu=None): EDLogging.__init__(self) self.pluginName = strPluginName self.startTime = time.time() try: self.iNbCpu = int(iNbCpu) except: self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs() self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu) self.__semaphoreNbThreads = Semaphore(self.iNbCpu) EDUtilsParallel.initializeNbThread(self.iNbCpu) self.jobQueue = Queue() self.processingSem = Semaphore() self.statLock = Semaphore() self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first" self.lastFailure = "No job Failed (yet)" self.lastSuccess = "No job succeeded (yet)"
def __init__(self): self.XML = "<XSDataInput>\ <normalizedImage><path><value>${FULLPATH}</value></path></normalizedImage>\ <correctedImage><path><value>${DIRDIRNAME}/2d/${BASENAME}.edf</value></path></correctedImage>\ <normalizedImageSize><value>4100000</value></normalizedImageSize>\ <integratedCurve><path><value>${DIRDIRNAME}/1d/${BASENAME}.edf</value></path></integratedCurve>\ <maskFile><path><value>${MASKFILE}</value></path></maskFile>\ <code><value>BSA</value></code>\ </XSDataInput>" self.maskfile = None self.dataFiles = [] self.wavelength = 1.0 self.debug = False self.mode = "offline" self.newerOnly = False self.nbcpu = multiprocessing.cpu_count() self.cpu_sem = Semaphore(self.nbcpu) self.process_sem = Semaphore() self.queue = Queue()
def __init__(self): """ Constructor of the main pure virtual class. This constructor implements: - the creation of the semaphore - definition of timer object (uninitialized as potentially not used) """ object.__init__(self) with self.__class__.__semaphoreId: self.__class__.__iId_class += 1 self.__iId = self.__class__.__iId_class self.__semaphore = Semaphore() self.__fTimeInit = None self.__fTimeEnd = None self.__classname = None
def initializeNbThread(cls, _iNbThread=None): """ Class method: Initialises a semaphore with the right number of threads @param _iNbThread: the maximum number of threads concurrently running and CPU intensive @type _iNbThread: integer """ with cls._semaphoreInit: if cls._semaphoreNbThreads is None: if _iNbThread is None: _iNbThread = cls.detectNumberOfCPUs() EDVerbose.DEBUG( "Initializing EDUtilsParallel semaphoreNbThreads to %s" % _iNbThread) cls._iNbThreads = _iNbThread cls._semaphoreNbThreads = Semaphore(_iNbThread) else: if cls._iNbThreads != _iNbThread: EDVerbose.WARNING( "cls._semaphoreNbThreads was already initialized to %s, not changing to %s" % (cls._iNbThreads, _iNbThread))
class EDPluginWaitFile(EDPlugin): """ Plugins that waits for a file to be written and reach a certain size """ #offers some extra seconds between officiel timeout and allows the plugin to finish gracefully. EXTRA_TIME = 5 DELTA_TIME = 1 DEFAULT_TIMEOUT = 2 config_timeout = None writeXmlInOut = True writeDataXMLOutput = True writeDataXMLInput = True sem = Semaphore() def __init__(self): """ """ EDPlugin.__init__(self) self.setXSDataInputClass(XSDataInputWaitFile) self.__filename = None self.__exists = None self.__filesize = None self.__expectedSize = None self.__timeout = self.DEFAULT_TIMEOUT def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginWaitFile.checkParameters") self.checkMandatoryParameters(self.getDataInput(), "Data Input is None") self.checkMandatoryParameters(self.getDataInput().getExpectedFile(), "Data Input is None") self.checkMandatoryParameters(self.getDataInput().getExpectedSize(), "Data Input is None") def configure(self): """ Configuration step of the plugin: mainly extend the timeout by 5 seconds to let the plugin finish. """ if self.__class__.config_timeout is None: with self.__class__.sem: if self.__class__.config_timeout is None: iTimeOut = self.config.get(EDPlugin.CONF_TIME_OUT, None) if iTimeOut is not None: self.DEBUG( "EDPlugin.configure: Setting time out to %d s from plugin configuration." % iTimeOut) self.__class__.config_timeout = iTimeOut else: self.__class__.config_timeout = self.getDefaultTimeOut( ) self.__class__.writeXmlInOut = bool( self.config.get(self.CONF_WRITE_XML_INPUT_OUTPUT, True)) self.__class__.writeDataXMLOutput = bool( self.config.get(self.CONF_WRITE_XML_OUTPUT, True)) self.__class__.writeDataXMLInput = bool( self.config.get(self.CONF_WRITE_XML_INPUT, True)) self.__timeout = self.__class__.config_timeout self.setTimeOut(self.__class__.config_timeout + EDPluginWaitFile.EXTRA_TIME) self.setWriteXMLInputOutput(self.writeXmlInOut) self.setWriteXMLOutput(self.writeDataXMLOutput) self.setWriteXMLInput(self.writeDataXMLInput) def preProcess(self, _edObject=None): EDPlugin.preProcess(self) self.DEBUG("EDPluginWaitFile.preProcess") self.__filename = self.getDataInput().getExpectedFile().getPath( ).getValue() self.__expectedSize = self.getDataInput().getExpectedSize().getValue() if self.getDataInput().getTimeOut(): self.__timeout = self.getDataInput().getTimeOut().getValue() self.setTimeOut(self.getDataInput().getTimeOut().getValue() + EDPluginWaitFile.EXTRA_TIME) def process(self, _edObject=None): EDPlugin.process(self) self.DEBUG("EDPluginWaitFile.process") self.DEBUG( "EDPluginWaitFile Plugin TimeOut is set to: %s, internal TimeOut is %s" % (self.getTimeOut(), self.__timeout)) self.setTimeInit() dirname = os.path.dirname(self.__filename) while self.getRunTime() < self.__timeout: if os.path.isdir(dirname): fd = os.open(dirname, os.O_RDONLY) os.fstat(fd) os.close(fd) if os.path.exists(self.__filename): self.__filesize = os.path.getsize(self.__filename) if self.__filesize >= self.__expectedSize: break time.sleep(EDPluginWaitFile.DELTA_TIME) self.setTimeEnd() def postProcess(self, _edObject=None): EDPlugin.postProcess(self) self.DEBUG("EDPluginWaitFile.postProcess: Waited for %.3f s" % self.getRunTime()) xsDataResult = XSDataResultWaitFile() if os.path.exists(self.__filename): xsDataFile = XSDataFile() xsDataFile.setPath(XSDataString(self.__filename)) xsDataResult.setActualFile(xsDataFile) xsDataResult.setActualSize( XSDataInteger(os.path.getsize(self.__filename))) xsDataResult.setTimedOut( XSDataBoolean(self.getRunTime() >= self.__timeout)) # Create some output data self.setDataOutput(xsDataResult)
class EDUtilsPath: """ This is a static utility class for handling of paths. """ EDNA_HOME = dirname(dirname(dirname(abspath(__file__)))) os.environ["EDNA_HOME"] = EDNA_HOME _EDNA_SITE = None _EDNA_USERTEMPFOLDER = None _EDNA_TESTIMAGES = None _EDNA_TESTS = None _EDNA_TESTDATA = None _USER = os.getenv("USER", "UndefindedUser") _TMPDIR = os.getenv("TMPDIR", tempfile.gettempdir()) _EDNA_PROJECTS = {} # key: project name. Value: path to the project root _EDNA_PLUGINCACHE = None __semaphore = Semaphore() @classmethod def appendListOfPaths(cls, _strPath, _listOfPaths): """ Appends a list of paths to a path, for example [ "..", "..", "..", "..", "src" ], and returns the absolute path. """ return abspath(os.path.join(_strPath, *_listOfPaths)) @classmethod def mergePath(cls, _strPath1, _strPath2): """ Merges two paths and returns the absolute path. Deprecated: please use 'os.path.join' instead """ EDVerbose.log( "EDUtilsPath.mergePath is deprecated, please use 'os.path.join' instead" ) strNewPath = os.path.join(_strPath1, _strPath2) return abspath(strNewPath) @classmethod def getAbsolutePath(cls, _strPath): """ Returns the absolute path. Deprecated: please use 'os.path.abspath' instead """ EDVerbose.log( "EDUtilsPath.getAbsolutePath is deprecated, please use 'os.path.abspath' instead" ) return abspath(_strPath) @classmethod def existPath(cls, _strPath): """ Checks if a folder exists. Deprecated: please use 'os.path.exists' instead """ EDVerbose.log( "EDUtilsPath.existPath is deprecated, please use 'os.path.exists' instead" ) return exists(_strPath) @classmethod def getEdnaHome(cls): """ Returns the EDNA_HOME variable from environment if not already defined """ return cls.EDNA_HOME @classmethod def setEdnaHome(cls, _strEdnaHome): """ Sets EDNA_HOME home """ cls.EDNA_HOME = _strEdnaHome @classmethod def getEdnaSite(cls): """ Returns the EDNA_SITE variable from environment """ # EDNA Site if (cls._EDNA_SITE == None): cls._EDNA_SITE = os.environ.get("EDNA_SITE") if (cls._EDNA_SITE is None): EDVerbose.warning( "EDUtilsPath.getEdnaSite: EDNA_SITE not set, using EDNA_SITE='default'" ) cls._EDNA_SITE = "default" else: EDVerbose.DEBUG( "EDUtilsPath.getEdnaSite: EDNA_SITE is set: %s" % cls._EDNA_SITE) return cls._EDNA_SITE @classmethod def setEdnaSite(cls, _strEdnaSite): """ Sets EDNA_SITE """ cls._EDNA_SITE = _strEdnaSite EDNA_SITE = classproperty(getEdnaSite, setEdnaSite) @classmethod def getCwd(cls): """ Returns the current directory. Deprecated: please use 'os.getcwd' instead """ EDVerbose.log( "EDUtilsPath.getCwd is deprecated, please use 'os.getcwd' instead") return os.getcwd() @classmethod def createFolder(cls, _strFolderName): """ Creates a folder (directory) if it doesn't already exists. This used to be deprecated but IS neverthless thread safe (see bug#681) """ EDVerbose.log("EDUtilsPath.createFolder: %s" % _strFolderName) with cls.__semaphore: if (not exists(_strFolderName)): os.makedirs(_strFolderName) @classmethod def getFileList(cls, _strPath="./", _strMask="*"): """ Returns a list of file and directory names given a path to a directory. """ if _strMask == "*": listFile = os.listdir(_strPath) else: strFind = os.path.join(_strPath, _strMask) listPath = glob.glob(strFind) listFile = [] for strPath in listPath: strPath = os.path.basename(strPath) listFile.append(strPath) return listFile @classmethod def getFolderName(cls, _str): """ Returns the name of a folder (directory) for a given path. Deprecated: please use 'os.path.dirname' instead """ return dirname(_str) @classmethod def getEdnaUserTempFolder(cls): """ Returns the name of a temporary folder that is unique for a given user. """ if cls._EDNA_USERTEMPFOLDER is None: if os.environ.has_key("EDNA_USERTEMPFOLDER"): cls._EDNA_USERTEMPFOLDER = os.environ["EDNA_USERTEMPFOLDER"] if not os.path.exists(cls._EDNA_USERTEMPFOLDER): EDVerbose.warning( "EDNA_USERTEMPFOLDER environment variable is set to %s put the directory does not exist!" % cls._EDNA_USERTEMPFOLDER) cls._EDNA_USERTEMPFOLDER = None elif (not os.access(cls._EDNA_USERTEMPFOLDER, os.W_OK)): EDVerbose.warning( "EDNA_USERTEMPFOLDER environment variable is set to %s put the directory cannot be accessed!" % cls._EDNA_USERTEMPFOLDER) cls._EDNA_USERTEMPFOLDER = None if cls._EDNA_USERTEMPFOLDER is None: strEdnaTempFileDir = tempfile.gettempdir() try: # Working on Windows and Linux: strUserName = getpass.getuser() except Exception: # Working on MacOS: strUserName = os.getlogin() bIsOk = False # Check that we have write access to this directory: if os.access(strEdnaTempFileDir, os.W_OK) and os.access( strEdnaTempFileDir, os.X_OK): cls._EDNA_USERTEMPFOLDER = os.path.join( strEdnaTempFileDir, "edna-%s" % strUserName) # Check that we have write access to this directory: if not os.path.exists(cls._EDNA_USERTEMPFOLDER): try: os.mkdir(cls._EDNA_USERTEMPFOLDER) except Exception: EDVerbose.WARNING( "Error when trying to create the directory %s" % cls._EDNA_USERTEMPFOLDER) if os.access(cls._EDNA_USERTEMPFOLDER, os.W_OK) and os.access( cls._EDNA_USERTEMPFOLDER, os.X_OK): bIsOk = True if not bIsOk: # We cannot use the edna-<user name> folder... EDVerbose.WARNING( "EDUtilsFile.getEdnaUserTempFolder: cannot access user temporary directory %s" % cls._EDNA_USERTEMPFOLDER) # Create temporary directory cls._EDNA_USERTEMPFOLDER = tempfile.mkdtemp(prefix="edna-") EDVerbose.WARNING( "Created temporary directory for this session: %s" % cls._EDNA_USERTEMPFOLDER) EDVerbose.WARNING( "If you would like to continue to use this directory for future sessions" ) EDVerbose.WARNING( "please set then environment variable EDNA_USERTEMPFOLDER to %s" % cls._EDNA_USERTEMPFOLDER) return cls._EDNA_USERTEMPFOLDER @classmethod def getEdnaPluginCachePath(cls): """This private method initializes the path to the plugin cache file""" if cls._EDNA_PLUGINCACHE is None: if "EDNA_PLUGINCACHE" in os.environ.keys(): cls._EDNA_PLUGINCACHE = os.environ["EDNA_PLUGINCACHE"] strDirName = os.path.dirname(cls._EDNA_PLUGINCACHE) if os.path.exists(cls._EDNA_PLUGINCACHE) and (not os.access( cls._EDNA_PLUGINCACHE, os.W_OK)): EDVerbose.warning( "EDNA_PLUGINCACHE environment variable is set to %s but the file is not writeable!" % cls._EDNA_PLUGINCACHE) cls._EDNA_PLUGINCACHE = None elif not os.path.exists(strDirName): EDVerbose.warning( "EDNA_PLUGINCACHE environment variable is set to %s put the parent directory does not exist!" % cls._EDNA_PLUGINCACHE) cls._EDNA_PLUGINCACHE = None elif not os.access(strDirName, os.W_OK): EDVerbose.warning( "EDNA_PLUGINCACHE environment variable is set to %s put the parent directory cannot be accessed!" % cls._EDNA_PLUGINCACHE) cls._EDNA_PLUGINCACHE = None if cls._EDNA_PLUGINCACHE is None: strTempFileDir = EDUtilsPath.getEdnaUserTempFolder() # We create a hash of the path in order to be able to reference several different EDNA_HOME # caches in the same directory strCacheFileName = hashlib.sha1( os.getenv('EDNA_HOME')).hexdigest() + ".xml" cls._EDNA_PLUGINCACHE = os.path.abspath( os.path.join(strTempFileDir, strCacheFileName)) EDVerbose.DEBUG("EDFactoryPlugin: Path to plugin cache: %s" % cls._EDNA_PLUGINCACHE) return cls._EDNA_PLUGINCACHE @classmethod def getEdnaTestPath(cls): """ This class method initializes the path to the test path (probably unused) """ if cls._EDNA_TESTS is None: if "EDNA_TESTS" in os.environ.keys(): cls.setEdnaTestDataPath(os.environ["EDNA_TESTS"]) if cls._EDNA_TESTS is None: cls._EDNA_TESTS = os.path.join( cls.EDNA_HOME, "tests", ) EDVerbose.DEBUG( "EDFactoryPlugin: Path to Test Data directory : %s" % cls._EDNA_TESTS) return cls._EDNA_TESTS @classmethod def setEdnaTestPath(cls, value): """ This class method initializes the path to the test data (probably unused) """ cls._EDNA_TESTS = os.path.abspath(value) if not os.path.isdir(cls._EDNA_TESTS): EDVerbose.warning( "EDNA_TESTS environment variable is set to %s put the directory does not exist!" % cls._EDNA_TESTS) cls._EDNA_TESTS = None EDNA_TESTS = classproperty(getEdnaTestPath, setEdnaTestPath) @classmethod def getEdnaTestDataPath(cls): """ This class method initializes the path to the test data (probably unused) """ if cls._EDNA_TESTDATA is None: if "EDNA_TESTDATA" in os.environ.keys(): cls.setEdnaTestDataPath(os.environ["EDNA_TESTDATA"]) if cls._EDNA_TESTDATA is None: cls._EDNA_TESTDATA = os.path.join(cls.EDNA_HOME, "tests", "data") EDVerbose.DEBUG( "EDFactoryPlugin: Path to Test Data directory : %s" % cls._EDNA_TESTDATA) return cls._EDNA_TESTDATA @classmethod def setEdnaTestDataPath(cls, value): """ This class method initializes the path to the test data (probably unused) """ cls._EDNA_TESTDATA = os.path.abspath(value) if not os.path.isdir(cls._EDNA_TESTDATA): EDVerbose.warning( "EDNA_TESTDATA environment variable is set to %s put the directory does not exist!" % cls._EDNA_TESTDATA) cls._EDNA_TESTDATA = None EDNA_TESTDATA = classproperty(getEdnaTestDataPath, setEdnaTestDataPath) @classmethod def getEdnaTestDataImagesPath(cls): """ This class method initializes the path to the test images downloaded from internet """ if cls._EDNA_TESTIMAGES is None: if "EDNA_TESTIMAGES" in os.environ.keys(): cls.setEdnaTestDataImagesPath(os.environ["EDNA_TESTIMAGES"]) if cls._EDNA_TESTIMAGES is None: cls._EDNA_TESTIMAGES = os.path.join(cls.EDNA_HOME, "tests", "data", "images") EDVerbose.DEBUG( "EDFactoryPlugin: Path to Temporary Images directory : %s" % cls._EDNA_TESTIMAGES) return cls._EDNA_TESTIMAGES @classmethod def setEdnaTestDataImagesPath(cls, value): """ This class method initializes the path to the test images downloaded from internet """ cls._EDNA_TESTIMAGES = os.path.abspath(value) if os.path.exists(cls._EDNA_TESTIMAGES) and (not os.access( cls._EDNA_TESTIMAGES, os.W_OK)): EDVerbose.warning( "EDNA_TESTIMAGES environment variable is set to %s but the file is not writeable!" % cls._EDNA_TESTIMAGES) cls._EDNA_TESTIMAGES = None elif not os.path.exists(cls._EDNA_TESTIMAGES): EDVerbose.warning( "EDNA_TESTIMAGES environment variable is set to %s put the directory does not exist!" % cls._EDNA_TESTIMAGES) cls._EDNA_TESTIMAGES = None elif not os.access(cls._EDNA_TESTIMAGES, os.W_OK): EDVerbose.warning( "EDNA_TESTIMAGES environment variable is set to %s put the directory cannot be accessed!" % cls._EDNA_TESTIMAGES) cls._EDNA_TESTIMAGES = None EDNA_TESTIMAGES = classproperty(getEdnaTestDataImagesPath, setEdnaTestDataImagesPath) @classmethod def getDictOfPaths(cls): """ Return an os.environ like dict enriched with internal path """ res = { "${EDNA_HOME}": cls.EDNA_HOME, "${EDNA_SITE}": cls.EDNA_SITE, "${EDNA_TESTS}": cls.EDNA_TESTS, "${EDNA_TESTDATA}": cls.EDNA_TESTDATA, "${EDNA_TESTIMAGES}": cls.EDNA_TESTIMAGES, "${USER}": cls._USER, "${TMPDIR}": cls._TMPDIR } return res
class EDObject(object): """ Virtual base class for all EDNA Objects (classes). It offers some synchronization and locking capabilities to make the code thread safe. """ __semaphoreId = Semaphore() __iId_class = 0 def __init__(self): """ Constructor of the main pure virtual class. This constructor implements: - the creation of the semaphore - definition of timer object (uninitialized as potentially not used) """ object.__init__(self) with self.__class__.__semaphoreId: self.__class__.__iId_class += 1 self.__iId = self.__class__.__iId_class self.__semaphore = Semaphore() self.__fTimeInit = None self.__fTimeEnd = None self.__classname = None def getId(self): return self.__iId def getClassName(self): """ Retrieves the name of the class @return: the name of the class @rtype: string """ return self.__class__.__name__ def synchronizeOn(self): """ This method must be used in together with the method synchronizeOff(). This method makes the code threadsafe till the method synchronizeOff is called. """ self.__semaphore.acquire() def synchronizeOff(self): """ This method must be used in together with the method synchronizeOn(). """ self.__semaphore.release() def locked(self): return self.__semaphore def setTimeInit(self): """ Initializes the timer for the object """ if self.__fTimeInit is None: self.__fTimeInit = time.time() def getTimeInit(self): """ Retrieves the time of initialization @return: number of seconds since epoch @rtype: float """ return self.__fTimeInit def setTimeEnd(self): """ Set the end of calculation time for the given object """ if self.__fTimeEnd is None: self.__fTimeEnd = time.time() def getTimeEnd(self): """ Retrieves the time of end of task @return: number of seconds since epoch @rtype: float """ return self.__fTimeEnd def getRunTime(self): """ @returns: the RunTime for the given object @rtype: float """ fRetrunRunTime = 0.0 if self.__fTimeInit is not None: if self.__fTimeEnd is None: fRetrunRunTime = time.time() - self.__fTimeInit else: fRetrunRunTime = self.__fTimeEnd - self.__fTimeInit return fRetrunRunTime
class EDPluginExecPyFAIv1_0(EDPluginExec): """ This is the basic plugin of PyFAI for azimuthal integration """ _dictGeo = {} #key:tuple(ai.param), value= ai _sem = Semaphore() def __init__(self): """ """ EDPluginExec.__init__(self) self.setXSDataInputClass(XSDataInputPyFAI) self.shared = None self.strOutputFile = None self.ai = None #this is the azimuthal integrator to use self.data = None self.mask = None self.nbPt = None self.dummy = None self.delta_dummy = None self.npaOut = None def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginExecPyFAIv1_0.checkParameters") self.checkMandatoryParameters(self.dataInput, "Data Input is None") # self.checkMandatoryParameters(self.dataInput.geometry, "No Geometry given") # self.checkMandatoryParameters(self.dataInput.detector, "No Detector given") self.checkMandatoryParameters(self.dataInput.input, "No input image given") self.checkMandatoryParameters(self.dataInput.nbPt, "No Number of points given for integration") @staticmethod def getDetector(xsDetector): detector = None if xsDetector.name and (xsDetector.name.value in dir(pyFAI.detectors)): detector = getattr(pyFAI.detectors, xsDetector.name.value)() else: pixel2 = EDUtilsUnit.getSIValue(xsDetector.pixelSizeX) pixel1 = EDUtilsUnit.getSIValue(xsDetector.pixelSizeY) if xsDetector.splineFile and os.path.isFile(xsDetector.splineFile.path.value): dictGeo = {"pixel1":pixel1, "pixel2":pixel2, "splineFile":xsDetector.splineFile.path.value} else: dictGeo = {"pixel1":pixel1, "pixel2":pixel2, "splineFile":None} detector = pyFAI.detectors.Detector() detector.setPyFAI(**dictGeo) return detector def preProcess(self, _edObject=None): EDPluginExec.preProcess(self) self.DEBUG("EDPluginExecPyFAIv1_0.preProcess") sdi = self.dataInput ai = pyFAI.AzimuthalIntegrator() if sdi.geometryFit2D is not None: xsGeometry = sdi.geometryFit2D detector = self.getDetector(xsGeometry.detector) d = {"direct": EDUtilsUnit.getSIValue(xsGeometry.distance) * 1000, #fit2D takes the distance in mm "centerX": xsGeometry.beamCentreInPixelsX.value , "centerY":xsGeometry.beamCentreInPixelsY.value , "tilt": xsGeometry.angleOfTilt.value, "tiltPlanRotation": xsGeometry.tiltRotation.value} d.update(detector.getFit2D()) ai.setFit2D(**d) elif sdi.geometryPyFAI is not None: xsGeometry = sdi.geometryPyFAI detector = self.getDetector(xsGeometry.detector) d = {"dist": EDUtilsUnit.getSIValue(xsGeometry.sampleDetectorDistance), "poni1": EDUtilsUnit.getSIValue(xsGeometry.pointOfNormalIncidence1), "poni2": EDUtilsUnit.getSIValue(xsGeometry.pointOfNormalIncidence2), "rot1": EDUtilsUnit.getSIValue(xsGeometry.rotation1), "rot2": EDUtilsUnit.getSIValue(xsGeometry.rotation2), "rot3": EDUtilsUnit.getSIValue(xsGeometry.rotation3)} d.update(detector.getPyFAI()) ai.setPyFAI(**d) else: strError = "Geometry definition in %s, not recognized as a valid geometry%s %s" % (sdi, os.linesep, sdi.marshal()) self.ERROR(strError) raise RuntimeError(strError) ######################################################################## # Choose the azimuthal integrator ######################################################################## with self.__class__._sem: if tuple(ai.param) in self.__class__._dictGeo: self.ai = self.__class__._dictGeo[tuple(ai.param)] else: self.__class__._dictGeo[tuple(ai.param)] = ai self.ai = ai self.data = EDUtilsArray.getArray(self.dataInput.input).astype(float) if sdi.dark is not None: self.data -= EDUtilsArray.getArray(sdi.dark) if sdi.flat is not None: self.data /= EDUtilsArray.getArray(sdi.flat) if sdi.mask is not None: self.mask = EDUtilsArray.getArray(sdi.mask) if sdi.wavelength is not None: self.ai.wavelength = EDUtilsUnit.getSIValue(sdi.wavelength) if sdi.output is not None: self.strOutputFile = sdi.output.path.value if sdi.dummy is not None: self.dummy = sdi.dummy.value if sdi.deltaDummy is not None: self.delta_dummy = sdi.deltaDummy.value if sdi.nbPt: self.nbPt = sdi.nbPt.value def process(self, _edObject=None): EDPluginExec.process(self) self.DEBUG("EDPluginExecPyFAIv1_0.process") data = EDUtilsArray.getArray(self.dataInput.input) if self.dataInput.saxsWaxs and self.dataInput.saxsWaxs.value.lower().startswith("s"): out = self.ai.saxs(self.data, nbPt=self.nbPt, filename=self.strOutputFile, mask=self.mask, dummy=self.dummy, delta_dummy=self.delta_dummy) else: out = self.ai.xrpd(self.data, nbPt=self.nbPt, filename=self.strOutputFile, mask=self.mask, dummy=self.dummy, delta_dummy=self.delta_dummy) self.npaOut = np.hstack((i.reshape(-1, 1) for i in out if i is not None)) def postProcess(self, _edObject=None): EDPluginExec.postProcess(self) self.DEBUG("EDPluginExecPyFAIv1_0.postProcess") # Create some output data if self.strOutputFile: output = XSDataImageExt(path=XSDataString(self.strOutputFile)) elif self.shared: output = XSDataImageExt(shared=XSDataString(self.shared)) else: output = XSDataImageExt(array=EDUtilsArray.arrayToXSData(self.npaOut)) xsDataResult = XSDataResultPyFAI(output=output) self.setDataOutput(xsDataResult)
class EDPluginBioSaxsHPLCv1_2(EDPluginControl): """ plugin for processing Saxs data coming from HPLC runs subsequently: *ProcessOneFile, *subtraction of buffer *AutoRg todo: only store references: Wait for flush to construct HDF5 file and (possibly) web pages with PNG graphs Changes since v1.0: * do not perfom Gnom not Porod analysis * return Summed Intensity and time stamp * fix issue with timestamp. Changes since v1.1: * Read configuration for a buffer threshold """ strControlledPluginProcessOneFile = "EDPluginBioSaxsProcessOneFilev1_4" strControlledPluginDatop = "EDPluginExecDatopv2_0" strControlledPluginAutoRg = "EDPluginExecAutoRgv1_0" strControlledPluginDatCmp = "EDPluginExecDatcmpv2_0" strControlledPluginDatAver = "EDPluginExecDataverv2_0" dictHPLC = {} # key=runId, value= HPLCrun instance _sem = Semaphore() SIMILARITY_THRESHOLD_SAMPLE_KEY = "similaritySample" SIMILARITY_THRESHOLD_BUFFER_KEY = "similarityBuffer" SIMILARITY_THRESHOLD_SAMPLE_DEFAULT = 0.0 SIMILARITY_THRESHOLD_BUFFER_DEFAULT = 0.1 SIMILARITY_THRESHOLD_SAMPLE = None SIMILARITY_THRESHOLD_BUFFER = None def __init__(self): """ """ EDPluginControl.__init__(self) self.setXSDataInputClass(XSDataInputBioSaxsHPLCv1_0) self.edPluginProcessOneFile = None self.edPluginSubtract = None self.edPluginAutoRg = None self.edPluginDatCmp = None self.xsDataResult = XSDataResultBioSaxsHPLCv1_0() self.runId = None self.frameId = None self.frame = None self.hplc_run = None self.curve = None self.subtracted = None self.lstExecutiveSummary = [] self.isBuffer = False self.scatter_data = None def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginBioSaxsHPLCv1_2.checkParameters") self.checkMandatoryParameters(self.dataInput, "Data Input is None") self.checkMandatoryParameters(self.dataInput.rawImage, "No raw image") self.checkMandatoryParameters(self.dataInput.sample, "no Sample parameter") self.checkMandatoryParameters(self.dataInput.experimentSetup, "No experimental setup parameter") def configure(self): """ Configures the HPLC plugin by reading from the configuration file - The threshold for similarity with sample & buffer """ EDPluginControl.configure(self) if self.__class__.SIMILARITY_THRESHOLD_SAMPLE is None: with self._sem: if self.__class__.SIMILARITY_THRESHOLD_SAMPLE is None: self.DEBUG("EDPluginBioSaxsHPLCv1_2.configure") self.__class__.SIMILARITY_THRESHOLD_BUFFER = float( self.config.get( self.SIMILARITY_THRESHOLD_BUFFER_KEY, self.SIMILARITY_THRESHOLD_BUFFER_DEFAULT)) self.__class__.SIMILARITY_THRESHOLD_SAMPLE = float( self.config.get( self.SIMILARITY_THRESHOLD_SAMPLE_KEY, self.SIMILARITY_THRESHOLD_SAMPLE_DEFAULT)) def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginBioSaxsHPLCv1_2.preProcess") sdi = self.dataInput if sdi.runId is not None: self.runId = sdi.runId.value else: path = sdi.rawImage.path.value if "_" in path: self.runId = path[::-1].split("_", 1)[1][::-1] else: self.runId = path with self._sem: if self.runId not in self.dictHPLC: self.dictHPLC[self.runId] = HPLCrun(self.runId) self.hplc_run = self.dictHPLC[self.runId] if sdi.frameId is not None: self.frameId = sdi.frameId.value else: path = sdi.rawImage.path.value if "_" in path: digits = os.path.splitext( os.path.basename(path))[0].split("_")[-1] try: self.frameId = int(digits) except ValueError: self.WARNING( "frameId is supposed to be an integer, I got %s" % digits) self.frameId = digits else: self.warning("using frameID=0 in tests, only") self.frameId = 0 with self._sem: self.frame = HPLCframe(self.runId, self.frameId) self.hplc_run.frames[self.frameId] = self.frame if sdi.bufferCurve and os.path.exists(sdi.bufferCurve.path.value): with self._sem: self.hplc_run.buffer = sdi.bufferCurve.path.value if self.hplc_run.hdf5_filename: hplc = self.hplc_run.hdf5_filename elif sdi.hplcFile: hplc = sdi.hplcFile.path.value else: path = sdi.rawImage.path.value dirname = os.path.dirname(os.path.dirname(path)) hplc_dir = os.path.join(dirname, "HPLC") if not os.path.exists(hplc_dir): try: os.mkdir(hplc_dir) except: pass # I don't care if "_" in path: hplc = "_".join( os.path.splitext( os.path.basename(path))[0].split("_")[:-1]) + ".h5" else: hplc = os.path.splitext(os.path.basename(path))[0] + ".h5" hplc = os.path.join(hplc_dir, hplc) if not self.hplc_run.hdf5_filename: with self._sem: self.hplc_run.init_hdf5(hplc) # self.xsDataResult.hplcFile = XSDataFile(XSDataString(hplc)) def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsHPLCv1_2.process") xsdIn = XSDataInputBioSaxsProcessOneFilev1_0( rawImage=self.dataInput.rawImage, sample=self.dataInput.sample, experimentSetup=self.dataInput.experimentSetup, rawImageSize=self.dataInput.rawImageSize, normalizedImage=self.dataInput.normalizedImage, integratedCurve=self.dataInput.integratedCurve, runId=self.dataInput.runId, frameId=self.dataInput.frameId) self.edPluginProcessOneFile = self.loadPlugin( self.strControlledPluginProcessOneFile) self.edPluginProcessOneFile.dataInput = xsdIn self.edPluginProcessOneFile.connectSUCCESS( self.doSuccessProcessOneFile) self.edPluginProcessOneFile.connectFAILURE( self.doFailureProcessOneFile) self.edPluginProcessOneFile.executeSynchronous() if self.isFailure(): return xsdIn = XSDataInputDatcmp(inputCurve=[ XSDataFile(XSDataString(self.hplc_run.first_curve)), XSDataFile(XSDataString(self.curve)) ]) self.edPluginDatCmp = self.loadPlugin(self.strControlledPluginDatCmp) self.edPluginDatCmp.dataInput = xsdIn self.edPluginDatCmp.connectSUCCESS(self.doSuccessDatCmp) self.edPluginDatCmp.connectFAILURE(self.doFailureDatCmp) self.edPluginDatCmp.executeSynchronous() if self.isFailure() or self.isBuffer: return if self.dataInput.subtractedCurve is not None: subtracted = self.dataInput.subtractedCurve.path.value else: subtracted = os.path.splitext(self.curve)[0] + "_sub.dat" if self.hplc_run.buffer is not None: xsdIn = XSDataInputDatop(inputCurve=[ XSDataFile(XSDataString(self.curve)), XSDataFile(XSDataString(self.hplc_run.buffer)) ], outputCurve=XSDataFile( XSDataString(subtracted)), operation=XSDataString("sub")) else: xsdIn = XSDataInputDatop(inputCurve=[ XSDataFile(XSDataString(self.curve)), XSDataFile(XSDataString(self.hplc_run.first_curve)) ], outputCurve=XSDataFile( XSDataString(subtracted)), operation=XSDataString("sub")) self.edPluginDatop = self.loadPlugin(self.strControlledPluginDatop) self.edPluginDatop.dataInput = xsdIn self.edPluginDatop.connectSUCCESS(self.doSuccessDatop) self.edPluginDatop.connectFAILURE(self.doFailureDatop) self.edPluginDatop.executeSynchronous() if self.subtracted and os.path.exists(self.subtracted): self.edPluginAutoRg = self.loadPlugin( self.strControlledPluginAutoRg) self.edPluginAutoRg.dataInput = XSDataInputAutoRg( inputCurve=[XSDataFile(XSDataString(self.subtracted))]) self.edPluginAutoRg.connectSUCCESS(self.doSuccessAutoRg) self.edPluginAutoRg.connectFAILURE(self.doFailureAutoRg) self.edPluginAutoRg.executeSynchronous() def postProcess(self, _edObject=None): """ after process """ EDPluginControl.postProcess(self) self.DEBUG("EDPluginBioSaxsHPLCv1_2.postProcess") if self.hplc_run.buffer: self.xsDataResult.bufferCurve = XSDataFile( XSDataString(self.hplc_run.buffer)) if self.curve: self.xsDataResult.integratedCurve = XSDataFile( XSDataString(self.curve)) if self.subtracted: self.xsDataResult.subtractedCurve = XSDataFile( XSDataString(self.subtracted)) def finallyProcess(self, _edObject=None): EDPluginControl.finallyProcess(self) executiveSummary = os.linesep.join(self.lstExecutiveSummary) self.xsDataResult.status = XSDataStatus( executiveSummary=XSDataString(executiveSummary)) self.dataOutput = self.xsDataResult if self.frame: self.frame.processing = False def average_buffers(self): """ Average out all buffers """ self.lstExecutiveSummary.append("Averaging out buffer files: " + ", ".join(self.hplc_run.for_buffer)) xsdIn = XSDataInputDataver(inputCurve=[ XSDataFile(XSDataString(i)) for i in self.hplc_run.for_buffer ]) if self.dataInput.bufferCurve: xsdIn.outputCurve = self.dataInput.bufferCurve else: xsdIn.outputCurve = XSDataFile( XSDataString( self.hplc_run.first_curve[::-1].split("_", 1)[1][::-1] + "_buffer_aver%02i.dat" % len(self.hplc_run.for_buffer))) self.edPluginDatAver = self.loadPlugin(self.strControlledPluginDatAver) self.edPluginDatAver.dataInput = xsdIn self.edPluginDatAver.connectSUCCESS(self.doSuccessDatAver) self.edPluginDatAver.connectFAILURE(self.doFailureDatAver) self.edPluginDatAver.executeSynchronous() def doSuccessProcessOneFile(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessProcessOneFile") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessProcessOneFile") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) output = _edPlugin.dataOutput if not output.integratedCurve: strErr = "Edna plugin ProcessOneFile did not produce integrated curve" self.ERROR(strErr) self.lstExecutiveSummary.append(strErr) self.setFailure() return self.curve = output.integratedCurve.path.value if not os.path.exists(self.curve): strErr = "Edna plugin ProcessOneFile: integrated curve not on disk !!" self.ERROR(strErr) self.lstExecutiveSummary.append(strErr) self.setFailure() return self.xsDataResult.integratedCurve = output.integratedCurve self.xsDataResult.normalizedImage = output.normalizedImage self.xsDataResult.dataI = output.dataI self.xsDataResult.dataQ = output.dataQ self.xsDataResult.dataStdErr = output.dataStdErr if output.experimentSetup and output.experimentSetup.timeOfFrame: startTime = output.experimentSetup.timeOfFrame.value else: try: startTime = float( fabio.openheader(self.dataInput.rawImage.path.value). header["time_of_day"]) except Exception: self.ERROR("Unable to read time_of_day in header of %s" % self.dataInput.rawImage.path.value) startTime = 0 with self._sem: if not self.hplc_run.first_curve: self.hplc_run.first_curve = self.curve self.hplc_run.start_time = startTime self.frame.curve = self.curve self.frame.time = startTime self.xsDataResult.timeStamp = XSDataTime( value=(startTime - self.hplc_run.start_time)) self.calcIntensity() def doFailureProcessOneFile(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureProcessOneFile") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureProcessOneFile") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) else: self.lstExecutiveSummary.append( "Edna plugin ProcessOneFile failed.") self.setFailure() def doSuccessDatop(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessDatop") self.retrieveSuccessMessages(_edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessDatop") if _edPlugin and _edPlugin.dataOutput: output = _edPlugin.dataOutput if output.status and output.status.executiveSummary: self.lstExecutiveSummary.append( output.status.executiveSummary.value) if output.outputCurve: self.subtracted = output.outputCurve.path.value if os.path.exists(self.subtracted): self.xsDataResult.subtractedCurve = output.outputCurve self.frame.subtracted = self.subtracted else: strErr = "Edna plugin datop did not produce subtracted file %s" % self.subtracted self.ERROR(strErr) self.lstExecutiveSummary.append(strErr) self.setFailure() def doFailureDatop(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureDatop") self.retrieveFailureMessages(_edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureDatop") strErr = "Edna plugin datop failed." if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) else: self.lstExecutiveSummary.append(strErr) self.ERROR(strErr) self.setFailure() def doSuccessAutoRg(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessAutoRg") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessAutoRg") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) try: rg = _edPlugin.dataOutput.autoRgOut[0] except: self.ERROR("AutoRg returned nothing !") else: if rg.rg: self.frame.Rg = rg.rg.value if rg.rgStdev: self.frame.Rg_Stdev = rg.rgStdev.value if rg.i0: self.frame.I0 = rg.i0.value if rg.i0Stdev: self.frame.I0_Stdev = rg.i0Stdev.value if rg.quality: self.frame.quality = rg.quality.value self.xsDataResult.autoRg = rg """ Calculate the invariants Vc and Qr from the Rambo&Tainer 2013 Paper, also the the mass estimate based on Qr for proteins """ if self.subtracted and os.path.exists(self.subtracted): self.subtracted_data = numpy.loadtxt(self.subtracted) if self.subtracted_data is not None and\ self.frame.Rg and self.frame.Rg_Stdev and self.frame.I0 and self.frame.I0_Stdev: dictRTI = RamboTainerInvariant(self.subtracted_data, self.frame.Rg, self.frame.Rg_Stdev, self.frame.I0, self.frame.I0_Stdev, rg.firstPointUsed.value) # {'Vc': vc[0], 'dVc': vc[1], 'Qr': qr, 'dQr': dqr, 'mass': mass, 'dmass': dmass} self.frame.Vc = dictRTI.get("Vc") self.frame.Vc_Stdev = dictRTI.get("dVc") self.frame.Qr = dictRTI.get("Qr") self.frame.Qr_Stdev = dictRTI.get("dQ") self.frame.mass = dictRTI.get("mass") self.frame.mass_Stdev = dictRTI.get("dmass") xsdRTI = XSDataRamboTainer( vc=XSDataDouble(self.frame.Vc), qr=XSDataDouble(self.frame.Qr), mass=XSDataDouble(self.frame.mass), dvc=XSDataDouble(self.frame.Vc_Stdev), dqr=XSDataDouble(self.frame.Qr_Stdev), dmass=XSDataDouble(self.frame.mass_Stdev)) self.xsDataResult.rti = xsdRTI def doFailureAutoRg(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureAutoRg") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureAutoRg") strErr = "Edna plugin AutoRg failed." if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) self.lstExecutiveSummary.append(strErr) else: self.lstExecutiveSummary.append(strErr) self.setFailure() def doSuccessDatCmp(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessDatCmp") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessDatCmp") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.fidelity: fidelity = _edPlugin.dataOutput.fidelity.value else: strErr = "No fidelity in output of datcmp" self.error(strErr) self.lstExecutiveSummary.append(strErr) # self.setFailure() fidelity = 0 if self.hplc_run.buffer is None: if fidelity > self.SIMILARITY_THRESHOLD_SAMPLE: self.isBuffer = True if fidelity > self.SIMILARITY_THRESHOLD_BUFFER: self.hplc_run.for_buffer.append(self.curve) else: self.average_buffers() elif fidelity > self.SIMILARITY_THRESHOLD_SAMPLE: self.isBuffer = True def doFailureDatCmp(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureDatCmp") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureDatCmp") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) else: self.lstExecutiveSummary.append("Edna plugin DatCmp failed.") self.setFailure() def doSuccessDatAver(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_2.doSuccessDatAver") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_2.doSuccessDatAver") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.outputCurve: bufferCurve = _edPlugin.dataOutput.outputCurve.path.value if os.path.exists(bufferCurve): with self._sem: self.hplc_run.buffer = bufferCurve else: strErr = "DatAver claimed buffer is in %s but no such file !!!" % bufferCurve self.ERROR(strErr) self.lstExecutiveSummary.append(strErr) self.setFailure() def doFailureDatAver(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_2.doFailureDatAver") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_2.doFailureDatAver") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) else: self.lstExecutiveSummary.append("Edna plugin DatAver failed.") self.setFailure() def calcIntensity(self): """ Calculate invarients like: Sum(I), and set output datastructure. """ self.scatter_data = numpy.loadtxt( self.xsDataResult.integratedCurve.path.value) self.frame.sum_I = self.scatter_data[:, 1].sum() self.xsDataResult.summedIntensity = XSDataDouble( value=self.frame.sum_I)
class EDPluginBioSaxsHPLCv1_0(EDPluginControl): """ plugin for processing Saxs data coming from HPLC runs subsequently: *ProcessOneFile, *subtraction of buffer *SaxsAnalysis todo: only store references: Wait for flush to construct HDF5 file and (possibly) web pages with PNG graphs """ strControlledPluginProcessOneFile = "EDPluginBioSaxsProcessOneFilev1_3" strControlledPluginDatop = "EDPluginExecDatopv1_0" strControlledPluginSaxsAnalysis = "EDPluginControlSaxsAnalysisv1_0" strControlledPluginDatCmp = "EDPluginExecDatcmpv1_0" strControlledPluginDatAver = "EDPluginExecDataverv1_0" dictHPLC = {} #key=runId, value= HPLCrun instance _sem = Semaphore() def __init__(self): """ """ EDPluginControl.__init__(self) self.setXSDataInputClass(XSDataInputBioSaxsHPLCv1_0) self.__edPluginProcessOneFile = None self.__edPluginSubtract = None self.__edPluginSaxsAnalysis = None self.__edPluginDatCmp = None self.xsDataResult = XSDataResultBioSaxsHPLCv1_0() self.runId = None self.frameId = None self.frame = None self.hplc_run = None self.curve = None self.subtracted = None self.lstExecutiveSummary = [] self.isBuffer = False def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginBioSaxsHPLCv1_0.checkParameters") self.checkMandatoryParameters(self.dataInput, "Data Input is None") self.checkMandatoryParameters(self.dataInput.rawImage, "No raw image") self.checkMandatoryParameters(self.dataInput.sample, "no Sample parameter") self.checkMandatoryParameters(self.dataInput.experimentSetup, "No experimental setup parameter") def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginBioSaxsHPLCv1_0.preProcess") sdi = self.dataInput if sdi.runId is not None: self.runId = sdi.runId.value else: path = sdi.rawImage.path.value if "_" in path: self.runId = path[::-1].split("_", 1)[1][::-1] else: self.runId = path with self._sem: if self.runId not in self.dictHPLC: self.dictHPLC[self.runId] = HPLCrun(self.runId) self.hplc_run = self.dictHPLC[self.runId] if sdi.frameId is not None: self.frameId = sdi.frameId.value else: path = sdi.rawImage.path.value if "_" in path: digits = os.path.splitext( os.path.basename(path))[0].split("_")[-1] try: self.frameId = int(digits) except ValueError: self.WARNING( "frameId is supposed to be an integer, I got %s" % digits) self.frameId = digits else: self.warning("using frameID=0 in tests, only") self.frameId = 0 with self._sem: self.frame = HPLCframe(self.runId, self.frameId) self.hplc_run.frames[self.frameId] = self.frame if sdi.bufferCurve and os.path.exists(sdi.bufferCurve.path.value): with self._sem: self.hplc_run.buffer = sdi.bufferCurve.path.value if self.hplc_run.hdf5_filename: hplc = self.hplc_run.hdf5_filename elif sdi.hplcFile: hplc = sdi.hplcFile.path.value else: path = sdi.rawImage.path.value if "_" in path: hplc = "_".join( os.path.splitext(path)[0].split("_")[:-1]) + ".h5" else: hplc = os.path.splitext(path)[0] + ".h5" if not self.hplc_run.hdf5_filename: with self._sem: self.hplc_run.init_hdf5(hplc) # self.xsDataResult.hplcFile = XSDataFile(XSDataString(hplc)) def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsHPLCv1_0.process") xsdIn = XSDataInputBioSaxsProcessOneFilev1_0( rawImage=self.dataInput.rawImage, sample=self.dataInput.sample, experimentSetup=self.dataInput.experimentSetup, rawImageSize=self.dataInput.rawImageSize, normalizedImage=self.dataInput.normalizedImage, integratedCurve=self.dataInput.integratedCurve, runId=self.dataInput.runId, frameId=self.dataInput.frameId) self.__edPluginProcessOneFile = self.loadPlugin( self.strControlledPluginProcessOneFile) self.__edPluginProcessOneFile.dataInput = xsdIn self.__edPluginProcessOneFile.connectSUCCESS( self.doSuccessProcessOneFile) self.__edPluginProcessOneFile.connectFAILURE( self.doFailureProcessOneFile) self.__edPluginProcessOneFile.executeSynchronous() if self.isFailure(): return xsdIn = XSDataInputDatcmp(inputCurve=[ XSDataFile(XSDataString(self.hplc_run.first_curve)), XSDataFile(XSDataString(self.curve)) ]) self.__edPluginDatCmp = self.loadPlugin(self.strControlledPluginDatCmp) self.__edPluginDatCmp.dataInput = xsdIn self.__edPluginDatCmp.connectSUCCESS(self.doSuccessDatCmp) self.__edPluginDatCmp.connectFAILURE(self.doFailureDatCmp) self.__edPluginDatCmp.executeSynchronous() if self.isFailure() or self.isBuffer: return if self.dataInput.subtractedCurve is not None: subtracted = self.dataInput.subtractedCurve.path.value else: subtracted = os.path.splitext(self.curve)[0] + "_sub.dat" xsdIn = XSDataInputDatop(inputCurve=[ XSDataFile(XSDataString(self.curve)), XSDataFile(XSDataString(self.hplc_run.first_curve)) ], outputCurve=XSDataFile( XSDataString(subtracted)), operation=XSDataString("sub")) self.__edPluginDatop = self.loadPlugin(self.strControlledPluginDatop) self.__edPluginDatop.dataInput = xsdIn self.__edPluginDatop.connectSUCCESS(self.doSuccessDatop) self.__edPluginDatop.connectFAILURE(self.doFailureDatop) self.__edPluginDatop.executeSynchronous() if self.subtracted and os.path.exists(self.subtracted): xsdIn = XSDataInputSaxsAnalysis( scatterCurve=XSDataFile(XSDataString(self.subtracted)), gnomFile=XSDataFile( XSDataString( os.path.splitext(self.subtracted)[0] + ".out"))) self.__edPluginSaxsAnalysis = self.loadPlugin( self.strControlledPluginSaxsAnalysis) self.__edPluginSaxsAnalysis.dataInput = xsdIn self.__edPluginSaxsAnalysis.connectSUCCESS( self.doSuccessSaxsAnalysis) self.__edPluginSaxsAnalysis.connectFAILURE( self.doFailureSaxsAnalysis) self.__edPluginSaxsAnalysis.executeSynchronous() def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginBioSaxsHPLCv1_0.postProcess") if self.hplc_run.buffer: self.xsDataResult.bufferCurve = XSDataFile( XSDataString(self.hplc_run.buffer)) if self.curve: self.xsDataResult.integratedCurve = XSDataFile( XSDataString(self.curve)) if self.subtracted: self.xsDataResult.subtractedCurve = XSDataFile( XSDataString(self.subtracted)) def finallyProcess(self, _edObject=None): EDPluginControl.finallyProcess(self) executiveSummary = os.linesep.join(self.lstExecutiveSummary) self.xsDataResult.status = XSDataStatus( executiveSummary=XSDataString(executiveSummary)) self.dataOutput = self.xsDataResult if self.frame: self.frame.processing = False def average_buffers(self): """ Average out all buffers """ self.lstExecutiveSummary.append("Averaging out buffer files: " + ", ".join(self.hplc_run.for_buffer)) xsdIn = XSDataInputDataver(inputCurve=[ XSDataFile(XSDataString(i)) for i in self.hplc_run.for_buffer ]) if self.dataInput.bufferCurve: xsdIn.outputCurve = self.dataInput.bufferCurve else: xsdIn.outputCurve = XSDataFile( XSDataString( self.hplc_run.first_curve[::-1].split("_", 1)[1][::-1] + "_buffer_aver%02i.dat" % len(self.hplc_run.for_buffer))) self.__edPluginDatAver = self.loadPlugin( self.strControlledPluginDatAver) self.__edPluginDatAver.dataInput = xsdIn self.__edPluginDatAver.connectSUCCESS(self.doSuccessDatAver) self.__edPluginDatAver.connectFAILURE(self.doFailureDatAver) self.__edPluginDatAver.executeSynchronous() def doSuccessProcessOneFile(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessProcessOneFile") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessProcessOneFile") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) output = _edPlugin.dataOutput if not output.integratedCurve: strErr = "Edna plugin ProcessOneFile did not produce integrated curve" self.ERROR(strErr) self.lstExecutiveSummary.append(strErr) self.setFailure() return self.curve = output.integratedCurve.path.value if not os.path.exists(self.curve): strErr = "Edna plugin ProcessOneFile: integrated curve not on disk !!" self.ERROR(strErr) self.lstExecutiveSummary.append(strErr) self.setFailure() return self.xsDataResult.integratedCurve = output.integratedCurve self.xsDataResult.normalizedImage = output.normalizedImage if output.experimentSetup and output.experimentSetup.timeOfFrame: startTime = output.experimentSetup.timeOfFrame.value else: startTime = None with self._sem: if not self.hplc_run.first_curve: self.hplc_run.first_curve = self.curve self.hplc_run.start_time = startTime self.frame.curve = self.curve self.frame.time = startTime def doFailureProcessOneFile(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureProcessOneFile") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureProcessOneFile") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) else: self.lstExecutiveSummary.append( "Edna plugin ProcessOneFile failed.") self.setFailure() def doSuccessDatop(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessDatop") self.retrieveSuccessMessages(_edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessDatop") if _edPlugin and _edPlugin.dataOutput: output = _edPlugin.dataOutput if output.status and output.status.executiveSummary: self.lstExecutiveSummary.append( output.status.executiveSummary.value) if output.outputCurve: self.subtracted = output.outputCurve.path.value if os.path.exists(self.subtracted): self.xsDataResult.subtractedCurve = output.outputCurve self.frame.subtracted = self.subtracted else: strErr = "Edna plugin datop did not produce subtracted file %s" % self.subtracted self.ERROR(strErr) self.lstExecutiveSummary.append(strErr) self.setFailure() def doFailureDatop(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureDatop") self.retrieveFailureMessages(_edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureDatop") strErr = "Edna plugin datop failed." if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) else: self.lstExecutiveSummary.append(strErr) self.ERROR(strErr) self.setFailure() def doSuccessSaxsAnalysis(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessSaxsAnalysis") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessSaxsAnalysis") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) gnom = _edPlugin.dataOutput.gnom if gnom: if gnom.rgGnom: self.frame.gnom = gnom.rgGnom.value if gnom.dmax: self.frame.Dmax = gnom.dmax.value if gnom.total: self.frame.total = gnom.total.value self.xsDataResult.gnom = gnom volume = _edPlugin.dataOutput.volume if volume: self.frame.volume = volume.value self.xsDataResult.volume = volume rg = _edPlugin.dataOutput.autoRg if rg: if rg.rg: self.frame.Rg = rg.rg.value if rg.rgStdev: self.frame.Rg_Stdev = rg.rgStdev.value if rg.i0: self.frame.I0 = rg.i0.value if rg.i0Stdev: self.frame.I0_Stdev = rg.i0Stdev.value if rg.quality: self.frame.quality = rg.quality.value self.xsDataResult.autoRg = rg def doFailureSaxsAnalysis(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureSaxsAnalysis") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureSaxsAnalysis") strErr = "Edna plugin SaxsAnalysis failed." if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) self.lstExecutiveSummary.append(strErr) else: self.lstExecutiveSummary.append(strErr) self.setFailure() def doSuccessDatCmp(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessDatCmp") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessDatCmp") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.fidelity: fidelity = _edPlugin.dataOutput.fidelity.value else: strErr = "No fidelity in output of datcmp" self.error(strErr) self.lstExecutiveSummary.append(strErr) #self.setFailure() fidelity = 0 if self.hplc_run.buffer is None: if fidelity > 0: self.isBuffer = True if fidelity > 0.1: self.hplc_run.for_buffer.append(self.curve) else: self.average_buffers() elif fidelity > 0: self.isBuffer = True def doFailureDatCmp(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureDatCmp") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureDatCmp") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) else: self.lstExecutiveSummary.append("Edna plugin DatCmp failed.") self.setFailure() def doSuccessDatAver(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_0.doSuccessDatAver") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_0.doSuccessDatAver") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.outputCurve: bufferCurve = _edPlugin.dataOutput.outputCurve.path.value if os.path.exists(bufferCurve): with self._sem: self.hplc_run.buffer = bufferCurve else: strErr = "DatAver claimed buffer is in %s but no such file !!!" % bufferCurve self.ERROR(strErr) self.lstExecutiveSummary.append(strErr) self.setFailure() def doFailureDatAver(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsHPLCv1_0.doFailureDatAver") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsHPLCv1_0.doFailureDatAver") if _edPlugin and _edPlugin.dataOutput and _edPlugin.dataOutput.status and _edPlugin.dataOutput.status.executiveSummary: self.lstExecutiveSummary.append( _edPlugin.dataOutput.status.executiveSummary.value) else: self.lstExecutiveSummary.append("Edna plugin DatAver failed.") self.setFailure()
from __future__ import with_statement __author__ = "Jérôme Kieffer" __license__ = "GPLv3+" __copyright__ = "2011 ESRF" __status__ = "production" __date__ = "20120917" import os, shutil from EDPluginControl import EDPluginControl from XSDataCommon import XSDataFile, XSDataString, XSDataStatus from XSDataEdnaSaxs import XSDataInputAutoSub, XSDataInputDataver, \ XSDataInputDatcmp, XSDataInputAutoRg, XSDataInputDatop from XSDataEdnaSaxs import XSDataResultAutoSub from EDThreading import Semaphore _sem = Semaphore() def copy(src, dst): """ Operating system independent copy that does symbolic links on posix system. This implements a lock to be extremely thread-safe as BsxCube call many times the same plugin ... at the same time """ with _sem: if os.path.exists(src): if os.name == "posix": if os.path.islink(dst): os.unlink(dst) os.symlink(src, dst) else:
class EDPluginExecAlignFramev1_0(EDPluginExec): """ An exec plugin that taked an images and a reference and alines the frame on the reference based on the SIFT algorithm. Derived from EDPluginExecMeasureOffsetv2_1.py """ use_sift_pyocl = None sift_keypoints = None sift_match = None keyindex = { } #key = md5 of the image; value = keypoint (as numpy recordarray) config_lock = Semaphore() def __init__(self): """ """ EDPluginExec.__init__(self) self.setXSDataInputClass(XSDataInputMeasureOffset) self.npaIm1 = None self.npaIm2 = None self.tOffset = None self.tCrop = None self.tCenter = None self.tWidth = None self.tSmooth = None self.bBackgroundsubtraction = False self.sobel = False def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginExecAlignFramev1_0.checkParameters") self.checkMandatoryParameters(self.dataInput, "Data Input is None") def configure(self): """ Configures the plugin (if not done yet """ EDPluginExec.configure(self) if self.use_sift_pyocl is None: with self.config_lock: if self.use_sift_pyocl is None: self.DEBUG("EDPluginExecAlignFramev1_0.configure") self.__class__.use_sift_pyocl = self.config.get( "sift_pyocl", False) print self.use_sift_pyocl, self.sift_keypoints, self.sift_match def calc_plan(self, image=None): if self.sift_keypoints is None: with self.config_lock: if self.sift_keypoints is None: if self.use_sift_pyocl and sift_pyocl: if self.config.get("device", None): self.__class__.sift_match = sift_pyocl.MatchPlan( device=self.config["device"]).match self.__class__.sift_keypoints = sift_pyocl.SiftPlan( template=image, device=self.config["device"]).keypoints else: deviceType = self.config.get("deviceType", "cpu") self.__class__.sift_match = sift_pyocl.MatchPlan( devicetype=deviceType).match self.__class__.sift_keypoints = sift_pyocl.SiftPlan( template=image, devicetype=deviceType).keypoints elif feature: self.__class__.sift_match = feature.sift_match self.__class__.sift_keypoints = feature.sift_keypoints else: strerr = "Impossible to make plan: use_sift_pyocl=%s but feature=%s and sift_pyocl=%s" % ( self.use_sift_pyoc, feature, sift_pyocl) self.ERROR(strerr) raise RuntimeError(strerr) print self.use_sift_pyocl, self.sift_keypoints, self.sift_match def preProcess(self, _edObject=None): EDPluginExec.preProcess(self) self.DEBUG("EDPluginExecAlignFramev1_0.preProcess") sdi = self.dataInput images = sdi.image arrays = sdi.array if len(images) == 2: self.npaIm1 = numpy.array(EDUtilsArray.getArray(images[0])) self.npaIm2 = numpy.array(EDUtilsArray.getArray(images[1])) elif len(arrays) == 2: self.npaIm1 = EDUtilsArray.xsDataToArray(arrays[0]) self.npaIm2 = EDUtilsArray.xsDataToArray(arrays[1]) else: strError = "EDPluginExecAlignFramev1_0.preProcess: You should either provide two images or two arrays, but I got: %s" % sdi.marshal( )[:1000] self.ERROR(strError) self.setFailure() raise RuntimeError(strError) crop = sdi.cropBorders if len(crop) > 1: self.tCrop = tuple([i.value for i in crop]) elif len(crop) == 1: self.tCrop = (crop[0].value, crop[0].value) center = sdi.center if len(center) > 1: self.tCenter = tuple([i.value for i in center]) elif len(center) == 1: self.tCenter = (center[0].value, center[0].value) width = sdi.width if len(width) > 1: self.tWidth = tuple([i.value for i in width]) elif len(width) == 1: self.tWidth = (width[0].value, width[0].value) smooth = sdi.smoothBorders if len(smooth) == 2: self.tSmooth = (smooth[0].value, smooth[1].value) elif len(smooth) == 1: self.tSmooth = (smooth[0].value, smooth[0].value) if sdi.backgroundSubtraction is not None: self.bBackgroundsubtraction = (sdi.backgroundSubtraction.value in [1, True, "true"]) if sdi.sobelFilter is not None: self.sobel = (sdi.sobelFilter in [1, True, "true"]) EDAssert.equal(self.npaIm1.shape, self.npaIm2.shape, "Images have the same size") def process(self, _edObject=None): EDPluginExec.process(self) self.DEBUG("EDPluginExecAlignFramev1_0.process") shape = self.npaIm1.shape if (self.tCrop is not None): d0min = min(shape[0], self.tCrop[0]) d1min = min(shape[1], self.tCrop[1]) d0max = max(0, shape[0] - self.tCrop[0]) d1max = max(0, shape[1] - self.tCrop[1]) shape = (d0max - d0min, d1max - d1min) else: d0min = 0 d0max = shape[0] d1min = 0 d1max = shape[1] if self.tCenter is None: #the default center is the geometry center of the image ... self.tCenter = [i // 2 for i in shape] if self.tWidth is not None: d0min = max(0, self.tCenter[0] - (self.tWidth[0] // 2)) d0max = min(shape[0], d0min + self.tWidth[0]) d1min = max(0, self.tCenter[1] - (self.tWidth[1] // 2)) d1max = min(shape[1], d1min + self.tWidth[1]) shape = (d0max - d0min, d1max - d1min) if shape != self.npaIm1.shape: self.DEBUG( "Redefining ROI to %s - %s ; %s - %s as crop=%s, center=%s and width=%s" % (d0min, d0max, d1min, d1max, self.tCrop, self.tCenter, self.tWidth)) #array contiguity is needed for checksum calculation self.npaIm1 = numpy.ascontiguousarray(self.npaIm1[d0min:d0max, d1min:d1max]) self.npaIm2 = numpy.ascontiguousarray(self.npaIm2[d0min:d0max, d1min:d1max]) shape = self.npaIm1.shape self.DEBUG("After Crop, images have shape : %s and %s " % (self.npaIm1.shape, self.npaIm2.shape)) if self.bBackgroundsubtraction: self.DEBUG("performing background subtraction") x = range(shape[0] - 1) + [shape[0] - 1] * (shape[0] - 1) + range( shape[0] - 1, 0, -1) + [0] * (shape[0] - 1) y = [0] * (shape[1] - 1) + range(shape[1] - 1) + [shape[1] - 1] * ( shape[1] - 1) + range(shape[1] - 1, 0, -1) spline1 = scipy.interpolate.SmoothBivariateSpline( x, y, [self.npaIm1[x[i], y[i]] for i in xrange(len(x))], kx=1, ky=1) spline2 = scipy.interpolate.SmoothBivariateSpline( x, y, [self.npaIm2[x[i], y[i]] for i in xrange(len(x))], kx=1, ky=1) self.npaIm1 -= spline1(range(shape[0]), range(shape[1])) self.npaIm2 -= spline2(range(shape[0]), range(shape[1])) if self.tSmooth is not None: self.npaIm1 *= MeasureOffset.make_mask(shape, self.tSmooth) self.npaIm2 *= MeasureOffset.make_mask(shape, self.tSmooth) if self.sobel: self.npaIm1 = scipy.ndimage.sobel(self.npaIm1) self.npaIm2 = scipy.ndimage.sobel(self.npaIm2) self.calc_plan(self.npaIm1) checksum1 = hashlib.md5(self.npaIm1).hexdigest() checksum2 = hashlib.md5(self.npaIm2).hexdigest() if checksum1 not in self.keyindex: self.keyindex[checksum1] = self.sift_keypoints(self.npaIm1) if checksum2 not in self.keyindex: self.keyindex[checksum2] = self.sift_keypoints(self.npaIm2) data = self.sift_match(self.keyindex[checksum1], self.keyindex[checksum2]) v0 = data[:, 0].y - data[:, 1].y v1 = data[:, 0].x - data[:, 1].x self.tOffset = [ XSDataDouble(numpy.median(v0)), XSDataDouble(numpy.median(v1)) ] def postProcess(self, _edObject=None): EDPluginExec.postProcess(self) self.DEBUG("EDPluginExecAlignFramev1_0.postProcess") # Create some output data self.setDataOutput(XSDataResultMeasureOffset(offset=self.tOffset))
class EDUtilsArray(object): """ This is a static utility class for handling numpy like arrays in XML. """ dTypeSize = { "int64": 8, "uint64": 8, "int32": 4, "uint32": 4, "int16": 2, "uint16": 2, "int8": 1, "uint8": 1, "float32": 4, "float64": 8 } dTypeFormat = { "int64": "q", "uint64": "Q", "int32": "l", "uint32": "L", "int16": "h", "uint16": "H", "int8": "b", "uint8": "B", "float32": "f", "float64": "d" } semArrayToXSData = Semaphore() semXsDataToArray = Semaphore() semGetArray = Semaphore() @classmethod def arrayToXSData(cls, _array, _bIncludeMd5sum=True, _bForceNoNumpy=False, _bUseAsserts=False): """ convert a numpy array or a list of list into an XSDataArray object @param _array: numpy array or array-like list @param _bIncludeMd5sum: should the md5sum be added to the XSDataArray instance. It is useful when sending object through the network It is a problem for testing @param _bForceNoNumpy: enables tests without numpy @type includeMd5sum: boolean @return: XSDataArray instance """ stringArray = "" shape = None dtype = None sizeDtype = None if bHaveNumpy is True and _bForceNoNumpy is False: EDVerbose.DEBUG("EDUtilsArray.arrayToXSData with numpy") # Enforce little Endianness if sys.byteorder == "big": _array.byteswap(True) stringArray = _array.tostring() shape = _array.shape dtype = str(_array.dtype) sizeDtype = len(numpy.zeros(1, dtype=_array.dtype).tostring()) else: EDVerbose.DEBUG("EDUtilsArray.arrayToXSData without numpy") sizeDtype = 8 # We enforce either double (float64) or int64 shape = [] subarray = _array while True: try: l = len(subarray) except TypeError: break shape.append(l) if l > 0: subarray = subarray[0] else: break if len(shape) == 1: if isinstance(_array[0], floatType): dtype = "float64" stringArray = struct.pack("<" + "d" * shape[0], *_array) else: dtype = "int64" stringArray = struct.pack("<" + "l" * shape[0], *_array) elif len(shape) == 2: if isinstance(_array[0][0], floatType): dtype = "float64" lineshape = "<" + "d" * shape[-1] else: dtype = "int64" lineshape = "<" + "q" * shape[-1] for subarray in _array: stringArray += struct.pack(lineshape, *subarray) elif len(shape) == 3: if isinstance(_array[0][0][0], floatType): dtype = "float64" lineshape = "<" + "d" * shape[-1] else: dtype = "int64" lineshape = "<" + "q" * shape[-1] for subarray in _array: for subsubarray in subarray: stringArray += struct.pack(lineshape, *subsubarray) else: EDVerbose.WARNING( "EDUtilsArray.arrayToXSDataArray: Array too large %s " % (shape)) size = 1 for i in shape: size *= i xsdArray = XSDataArray(data=base64.b64encode(stringArray), coding=XSDataString("base64"), shape=list(shape), dtype=dtype, size=size) if _bUseAsserts: EDAssert.equal(size * sizeDtype, len(stringArray), "string representing the array has the right size") if _bIncludeMd5sum is True: xsdArray.setMd5sum( XSDataString(hashlib.md5(stringArray).hexdigest())) return xsdArray @classmethod def xsDataToArray(cls, _xsdata, _bCheckMd5sum=True, _bForceNoNumpy=False, _bUseAsserts=False): """ convert a XSDataArray into either a numpy array or a list of list @param _xsdata: XSDataArray instance @param checkMd5sum: Check if the data are correct based on the checksum It is useful when sending object through the network It is a problem for testing @type _bCheckMd5sum: boolean @param _bForceNoNumpy: enables tests without numpy @type _bForceNoNumpy: boolean @return: numpy array or array-like list depending on what is available on the computer """ shape = tuple(_xsdata.getShape()) encData = _xsdata.getData() iSize = _xsdata.getSize() dtype = str(_xsdata.getDtype()) if _xsdata.getCoding() is not None: strCoding = _xsdata.getCoding().getValue() if strCoding == "base64": decData = base64.b64decode(encData) elif strCoding == "base32": decData = base64.b32decode(encData) elif strCoding == "base16": decData = base64.b16decode(encData) else: EDVerbose.WARNING( "Unable to recognize the encoding of the data !!! got %s, expected base64, base32 or base16, I assume it is base64 " % strCoding) decData = base64.b64decode(encData) else: EDVerbose.WARNING("No coding provided, I assume it is base64 ") strCoding = "base64" decData = base64.b64decode(encData) EDVerbose.DEBUG( "Reading numpy array: len(EncData)= %s, len(decData)=%s, shape=%s, size=%s, dtype=%s, coding=%s" % (len(encData), len(decData), shape, _xsdata.getSize(), _xsdata.getDtype(), strCoding)) if _bUseAsserts: EDAssert.equal(len(decData), cls.dTypeSize[dtype] * iSize, "decoded data has the expected size") if (_xsdata.getMd5sum() is not None) and (_bCheckMd5sum is True) and _bUseAsserts: EDAssert.equal(_xsdata.getMd5sum().getValue(), hashlib.md5(decData).hexdigest(), "md5sum is correct") if bHaveNumpy is True and _bForceNoNumpy is False: EDVerbose.DEBUG("EDUtilsArray.xsDataToArray with numpy") try: matIn = numpy.fromstring(decData, dtype=_xsdata.getDtype()) except Exception: matIn = numpy.fromstring(decData, dtype=numpy.dtype( str(_xsdata.getDtype()))) arrayOut = matIn.reshape(shape) # Enforce little Endianness if sys.byteorder == "big": arrayOut.byteswap(True) else: EDVerbose.DEBUG("EDUtilsArray.xsDataToArray without numpy") arrayOut = [] lineshape = "<" + cls.dTypeFormat[dtype] * shape[-1] if len(shape) == 1: arrayOut = list(struct.unpack(lineshape, decData)) elif len(shape) == 2: linesize = shape[-1] * cls.dTypeSize[dtype] for i in xrange(shape[0]): arrayOut.append( list( struct.unpack( lineshape, decData[i * linesize:(i + 1) * linesize]))) elif len(shape) == 3: linesize = shape[-1] * cls.dTypeSize[dtype] imgsize = shape[-2] * linesize for i in xrange(shape[0]): subarray = [] for j in xrange(shape[1]): subarray.append( list( struct.unpack( lineshape, decData[imgsize * i + j * linesize:imgsize * i + (j + 1) * linesize]))) arrayOut.append(subarray) return arrayOut @classmethod def getArray(cls, _inputObject): """ Tries to retrieve a numpy array from an Image or from a shared array or an XSD object As this method returns a numpy array, this method works only on plateform where numpy is available. @param _inputObject: XSDataArray or XSDataImageExt or XSDataFile @return: numpy ndarray """ npaOutput = None bError = False lstAttrObj = dir(_inputObject) if "array" in lstAttrObj and _inputObject.array is not None: npaOutput = cls.xsDataToArray(_inputObject.array) elif "shared" in lstAttrObj and _inputObject.shared is not None: npaOutput = EDShare[_inputObject.shared.value] elif "path" in lstAttrObj and (_inputObject.path is not None) and os.path.isfile( _inputObject.path.value): if bHaveFabio is True: try: npaOutput = fabio.open(_inputObject.path.value).data except Exception: bError = True else: bError = True elif "XSDataArray" in str(type(_inputObject)): npaOutput = cls.xsDataToArray(_inputObject) if bError is True: EDVerbose.ERROR( "EDUtilsArray.getArray works better on platform with numpy & fabio ... No solution found for you, sorry.%s%s " % (os.linesep, _inputObject.marshal())) return npaOutput
def __init__(self, runId, first_curve=None): self.id = runId self.buffer = None self.first_curve = first_curve self.frames = {} #key: id, value: HPLCframe instance self.curves = [] self.for_buffer = [] self.hdf5_filename = None self.hdf5 = None self.chunk_size = 250 self.lock = Semaphore() if first_curve: self.files.append(first_curve) self.max_size = None self.start_time = None self.time = None self.gnom = None self.Dmax = None self.total = None self.volume = None self.Rg = None self.Rg_Stdev = None self.I0 = None self.I0_Stdev = None self.quality = None self.q = None self.size = None self.scattering_I = None self.scattering_Stdev = None self.subtracted_I = None self.subtracted_Stdev = None self.sum_I = None self.Vc = None self.Qr = None self.mass = None self.Vc_Stdev = None self.Qr_Stdev = None self.mass_Stdev = None self.buffer_frames = None self.merge_frames = None self.buffer_I = None self.buffer_Stdev = None self.merge_I = None self.merge_Stdev = None self.merge_curves = [] self.merge_Rg = {} self.merge_analysis = {} self.merge_framesDIC = {} self.keys1d = [ "gnom", "Dmax", "total", "volume", "Rg", "Rg_Stdev", "I0", "I0_Stdev", "quality", "sum_I", "Vc", "Qr", "mass", "Vc_Stdev", "Qr_Stdev", "mass_Stdev" ] self.keys2d = [ "scattering_I", "scattering_Stdev", "subtracted_I", "subtracted_Stdev" ] self.keys_frames = ["buffer_frames", "merge_frames"] self.keys_merges = [ "buffer_I", "buffer_Stdev", "merge_I", "merge_Stdev" ]
class EDObject(object): """ Virtual base class for all EDNA Objects (classes). It offers some synchronization and locking capabilities to make the code thread safe. """ __semaphoreId = Semaphore() __iId_class = 0 profiling = {} def __init__(self): """ Constructor of the main pure virtual class. This constructor implements: - the creation of the semaphore - definition of timer object (uninitialized as potentially not used) """ object.__init__(self) with self.__class__.__semaphoreId: self.__class__.__iId_class += 1 self.__iId = self.__class__.__iId_class if self.__class__.__name__ not in self.__class__.profiling: self.__class__.profiling[self.__class__.__name__] = [] self.__semaphore = Semaphore() self.__fTimeInit = None self.__fTimeEnd = None self.__classname = None def getId(self): return self.__iId def getClassName(self): """ Retrieves the name of the class @return: the name of the class @rtype: string """ return self.__class__.__name__ def synchronizeOn(self): """ This method must be used in together with the method synchronizeOff(). This method makes the code thread-safe till the method synchronizeOff is called. """ self.__semaphore.acquire() def synchronizeOff(self): """ This method must be used in together with the method synchronizeOn(). """ self.__semaphore.release() def getSemaphoreValue(self): """ This method should only be used for debugging purpose... @return: the "internal" value of the semaphore @rtype: integer """ iValue = self.__semaphore._Semaphore__value # EDVerbose.WARNING("DEBUG INFO: The value of semaphore for instance of class %s with hash %s is %i" % (self.getClassName(), hash(self), iValue)) return iValue def locked(self): return self.__semaphore def setTimeInit(self): """ Initializes the timer for the object """ if self.__fTimeInit is None: self.__fTimeInit = time.time() def getTimeInit(self): """ Retrieves the time of initialization @return: number of seconds since epoch @rtype: float """ return self.__fTimeInit def setTimeEnd(self): """ Set the end of calculation time for the given object """ if self.__fTimeEnd is None: self.__fTimeEnd = time.time() if self.__fTimeInit: self.__class__.profiling[self.getClassName()].append(self.__fTimeEnd - self.__fTimeInit) def getTimeEnd(self): """ Retrieves the time of end of task @return: number of seconds since epoch @rtype: float """ return self.__fTimeEnd def getRunTime(self): """ @returns: the RunTime for the given object @rtype: float """ fRetrunRunTime = 0.0 if self.__fTimeInit is not None: if self.__fTimeEnd is None: fRetrunRunTime = time.time() - self.__fTimeInit else: fRetrunRunTime = self.__fTimeEnd - self.__fTimeInit return fRetrunRunTime @classmethod def analyze_profiling(cls): "Analyse the profiling an return a list of strings to be printed out" res = ["Analysis on: %s" % time.asctime(), " Calls | Mean (s) | Std dev | Total (s) | Plugin name", "-" * 80] subres = {} import numpy for name, lst_timimgs in cls.profiling.items(): npd = numpy.array(lst_timimgs) tot = npd.sum() line = " %6d | %8.3f | %8.3f | %9.3f | %s " % \ (npd.size, tot / npd.size, npd.std(), tot , name) subres[tot] = line timimgs = list(subres.keys()) timimgs.sort() for key in timimgs: res.append(subres[key]) return res
class EDUtilsPlatform(object): """ Static class for guessing: platform specific stuff. """ __semaphore = Semaphore() __semaphore.acquire() __strCmdSep = None __strCmdEnv = None __strEscapedLineSep = None __strLineSep = os.linesep __strSep = os.sep __strAltSep = os.altsep __strExtSep = os.extsep __strPathSep = os.pathsep __strName = os.name __strStartScript = None __PythonPlatformSize = int( round(math.log(sys.maxint + 1) / math.log(2) + 1)) __SystemPlatform = distutils.util.get_platform() if __PythonPlatformSize == 32 and __SystemPlatform.endswith("x86_64"): __PythonPlatform = __SystemPlatform[:-6] + "i386" else: __PythonPlatform = __SystemPlatform __PythonArchitecture = "lib.%s-%i.%i" % ( __PythonPlatform, sys.version_info[0], sys.version_info[1]) if sys.maxunicode == 65535: __PythonArchitecture += "-ucs2" __SystemArchitecture = "lib.%s-%i.%i" % ( __SystemPlatform, sys.version_info[0], sys.version_info[1]) if os.name == "java": # Then we are running under Jython ! from java.lang import System as javasystem if javasystem.getProperty("os.name").startswith("Windows"): __strName = "nt" else: __strName = "posix" if __strName == "nt": __strCmdSep = "&" __strCmdEnv = "set" __strEscapedSep = "/" __strEscapedLineSep = "\\r\\n" __strStartScript = "@ECHO OFF" elif __strName == "posix": __strCmdSep = ";" __strCmdEnv = "env" __strEscapedSep = "\\\\" __strEscapedLineSep = "\\n" __strStartScript = "#!%s" % sys.executable __semaphore.release() @classmethod def getArchitecture(cls): """ Returns the name of the architecture including the CPU arch and the version of python for the actual running python (i.e. can be i386 on a x86_64 computer) @return: lib-$OS-$arch-$PyVersion @rtype: python string """ return cls.__PythonArchitecture architecture = classproperty(getArchitecture) @classmethod def getSystemArchitecture(cls): """ Returns the name of the architecture including the CPU arch and the version of python for the actual operating system @return: lib-$OS-$arch-$PyVersion @rtype: python string """ return cls.__SystemArchitecture systemArchitecture = classproperty(getSystemArchitecture) @classmethod def getName(cls): """ Returns the name of the architecture @return: os.name like architecture (posix, nt, ... but never java even under jython) @rtype: python string """ return cls.__strName name = classproperty(getName) @classmethod def getCmdSep(cls): """ Returns the Command separator like - Under Unix: cmd1;cmd2 - Under Windows cmd1 & cmd2 @return: "&" or ";" depending on the architecture @rtype: python string """ return cls.__strCmdSep cmdSep = classproperty(getCmdSep) @classmethod def getCmdEnv(cls): """ @return: "env" or "set" to print the environment under the current operating system """ return cls.__strCmdEnv cmdEnv = classproperty(getCmdEnv) @classmethod def getEscapedSep(cls): """ return os.sep with "\" escaped """ return cls.__strEscapedSep escapedSep = classproperty(getEscapedSep) @classmethod def getEscapedLineSep(cls): """ return os.linesep with "\" escaped """ return cls.__strEscapedLineSep escapedLinesep = classproperty(getEscapedLineSep) @classmethod def getLineSep(cls): """ @return os.linesep """ return cls.__strLineSep linesep = classproperty(getLineSep) @classmethod def getSep(cls): """ @return: os.sep """ return cls.__strSep sep = classproperty(getSep) @classmethod def getStartScript(cls): """ @return: header for writing scripts under the current the archtecture (#!/usr/bin/python under linux) """ return cls.__strStartScript startScript = classproperty(getStartScript) @classmethod def getSize(cls): """ @return: the size of the environment, probably 32 or 64 bits """ return cls.__PythonPlatformSize size = classproperty(getSize) @classmethod def getSystemPlatform(cls): """ @return: linux-x86_64, if python 32bits is running under amd64 OS. """ return cls.__SystemPlatform systemPlatform = classproperty(getSystemPlatform) @classmethod def getPythonPlatform(cls): """ @return: linux-i386, if python 32bits is running under amd64 OS. """ return cls.__PythonPlatform pythonPlatform = classproperty(getPythonPlatform) @classmethod def getAltSep(cls): """ @return: os.altsep """ return cls.__strAltSep altsep = classproperty(getAltSep) @classmethod def getExtSep(cls): """ @return: "." as extension separator """ return cls.__strExtSep extsep = classproperty(getExtSep) @classmethod def getPathSep(cls): """ same as os.pathsep @return "/" under linux and "\" under windows. """ return cls.__strPathSep pathsep = classproperty(getPathSep) @classmethod def escape(cls, _string): """ escape \ (i.e. "\\") under windows as \\ (i.e. "\\\\") @return escaped string suitable for metaprogramming """ return _string.replace("\\", "\\\\") @classmethod def Popen(cls, *args, **kwargs): """ implementation of a platform independent subprocess.Popen method @return: subporcess.Popen instance. """ if os.name == "posix": #python under unix kwargs["preexec_fn"] = os.setsid return subprocess.Popen(*args, **kwargs) @classmethod def kill(cls, _iPid): """ implementation of a platform independent kill method @param _iPid: process ID @type _iPid: integer """ EDVerbose.log("EDUtilsPlatorm.kill called on PID: %s" % _iPid) if os.name == "posix": #python under unix os.killpg(_iPid, signal.SIGKILL) elif cls.architecture == "posix": #jython running under unix os.kill(_iPid, signal.SIGKILL) else: #windows, ... Nota: this only works from python2.7 under windows ! EDVerbose.WARNING("Kill Called to PID= %s with signal %s" % (_iPid, signal.SIGTERM)) os.kill(_iPid, signal.SIGTERM)
class EDJob(EDLogging): """ Create a module called EDJob * Most of what was done up to 09-2010 in EDParallelExecute should be done here * Each instance will be a job * Constructor takes a plugin name * Each instance will have taking an "setDataInput" method getting an XMLin (as string) * Each instance will gave a "getDataOutput" method with optional join * there could be a "join" method, waiting for the job to finish * Each instance will have a "execute" method and returning a JobId * Each instance will have a "setCallBack" method that stores the name of the external callback * provide status of a job * keep track of all plugin status * leave the time to plugin to initialize * static class retrieve job-instance, status, small-log ... * prevent multiple run of a single job ? * does not manage workload of the computer, should be managed at the ExecPlugin level Used for the tango binding, EDParallelExecute ... == class variables == dictPluginStatus[pluginName] = ["uninitialized"|"running"|"executed"| "failed"] dictJobs [JobId] = EDJob.Instance == static methods == getJob(JobId) """ PLUGIN_STATE_UNITIALIZED = "uninitialized" PLUGIN_STATE_RUNNING = "running" PLUGIN_STATE_SUCCESS = "success" PLUGIN_STATE_FAILURE = "failure" __edFactoryPlugin = EDFactoryPlugin() __dictJobs = {} __semaphore = Semaphore() __fStartTime = time.time() def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDLogging.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__pathXSDInput = None self.__pathXSDOutput = None self.__bXmlInputSet = False self.__status = None self.__name = None self.__runtime = None self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin( self.__strPluginName) if self.__edPlugin is None: raise RuntimeError("Unable to create plugin %s" % self.__strPluginName) self.__jobId = "%s-%08i" % (self.__strPluginName, self.__edPlugin.getId()) with self.__class__.__semaphore: self.__class__.__dictJobs[self.__jobId] = self if (self.__edPlugin is None): self.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the job (plugin) input data. @param: _oDataInput: could be either an String XML or an XSData object. @param _strDataInputKey: the key of an input data dictionnary The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ if _oDataInput in ["", None]: self.__bXmlInputSet = False return else: with self.locked(): if (self.__edPlugin is not None): self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey) self.__bXmlInputSet = True else: self.WARNING( "Setting DataInput for uninstanciated plugin %s." % self.__strPluginName) def getDataInput(self, _strDataInputKey=None): """ Returns the Plugin Input Data for a particular key. If the key is not provided a default key is used. """ if (self.__edPlugin is not None): return self.__edPlugin.getDataInput(_strDataInputKey).marshal() elif (self.__pathXSDInput is not None): return open(self.__pathXSDInput).read() else: self.WARNING("Getting DataInput for uninstanciated plugin %s." % self.__strPluginName) dataInput = property(getDataInput, setDataInput) def getDataOutput(self, _strDataOutputKey=None, _bWait=True): """ Returns the Plugin Output Data @param _bWait: shall we wait for the plugin to finish to retrieve output data: Yes by default. @type _bWait: boolean """ if _bWait: # Wait for plugin to finish befor returning data output self.synchronize() if (self.__edPlugin is not None): return self.__edPlugin.getDataOutput(_strDataOutputKey).marshal() elif self.__pathXSDOutput is not None: return open(self.__pathXSDOutput).read() else: self.WARNING( "Getting DataOutput for uninstanciated plugin or plugin has been garbage collected or output data were not saved. JobID was %s ." % self.__jobId) dataOutput = property(getDataOutput) def execute(self): """ Launch the EDNA plugin @return: JobId @rtype: string """ if not self.__bXmlInputSet: self.WARNING("Not executing job %s as input is empty" % self.__jobId) if (self.__edPlugin is not None): with self.locked(): self.__edPlugin.connectSUCCESS(self.successPluginExecution) self.__edPlugin.connectFAILURE(self.failurePluginExecution) self.__status = EDJob.PLUGIN_STATE_RUNNING self.__edPlugin.execute() return self.__jobId else: self.WARNING("Trying to run a plugin that does not exist: %s " % self.__strPluginName) def synchronize(self): """ Synchronize the execution of the job with the calling thread. """ with self.locked(): strStatus = self.__status if strStatus == EDJob.PLUGIN_STATE_RUNNING: self.__edPlugin.synchronize() elif strStatus == EDJob.PLUGIN_STATE_UNITIALIZED: self.WARNING("Unable to synchronize %s jobs" % strStatus) else: self.DEBUG("Unable to synchronize %s jobs" % strStatus) @classmethod def synchronizeAll(cls): """ Wait for all jobs to finish. """ EDVerbose.DEBUG("EDJob.synchronizeAll class method ") listJob = cls.__dictJobs.keys() for jobid in listJob: job = cls.__dictJobs[jobid] job.synchronize() if len(cls.__dictJobs) != len(listJob): EDVerbose.WARNING( "EDJob.synchronizeAll: New jobs have been launched while synchronizing" ) def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ with self.locked(): self.__status = EDJob.PLUGIN_STATE_SUCCESS self.screen("Plugin %s: success after %.3fs" % (self.__jobId, _edObject.getRunTime())) try: self.__edSlotSUCCESS.call(self.__jobId) except Exception: self.ERROR("Error in execution of Success call-back for %s" % self.__jobId) self.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: self.ERROR( "Error in execution of Common call-back (after success) for %s" % self.__jobId) self.writeErrorTrace() def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ with self.locked(): self.__status = EDJob.PLUGIN_STATE_FAILURE self.screen("Plugin %s: failure after %.3fs" % (self.__jobId, _edObject.getRunTime())) try: self.__edSlotFAILURE.call(self.__jobId) except Exception: self.ERROR("Error in execution of Failure call-back for %s" % self.__jobId) self.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: self.ERROR( "Error in execution of Common call-back (after failure) for %s" % self.__jobId) self.writeErrorTrace() def connectSUCCESS(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotSUCCESS.connect(_oMethod) def connectFAILURE(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotFAILURE.connect(_oMethod) def connectCallBack(self, _oMethod): """ @param _oMethod: function or method to be called - back """ with self.locked(): if (_oMethod != None): self.__edSlotCallBack.connect(_oMethod) def getJobId(self): """ @return: JobId i.e. EDPluginName-Number @rtype: string """ return self.__jobId jobId = property(getJobId, "EDJob.jobId: read-only property") getJobID = getJobId def getPluginName(self): """ @return: Name of the plugin @rtype: string """ return self.__strPluginName pluginName = property(getPluginName, "EDJob.pluginName: read-only property") def getPlugin(self): """ @return: the plugin (instance) @rtype: python object """ return self.__edPlugin plugin = property(getPlugin, "EDJob.plugin: read-only property") def getStatus(self): """ @return: status of the Job @rtype: string """ return self.__status status = property(getStatus, "EDJob.status: read-only property") def getName(self): return self.__name def setName(self, _strName): if self.__name is None: self.__name = _strName else: self.WARNING("EDJob.setName: One cannot rename a Job !!!") name = property(getName, setName, "EDJob.name: nickname of the job") def getMemSize(self): """ try to guess the size in memory of a job @return: expected size in memory """ if asizeof is not None: return asizeof.asizeof(self) @classmethod def getStatusFromID(cls, jobId): """ Retrieve the job (hence the plugin) status @param jobId: the Job identification number @type jobId: string @return: the EDJob status @rtype: string """ if jobId in cls.__dictJobs: strRet = cls.__dictJobs[jobId].getStatus() else: strRet = "Unable to retrieve such job: %s" % jobId EDVerbose.WARNING(strRet) return strRet getStatusFromId = getStatusFromID @classmethod def getJobFromID(cls, jobId): """ Retrieve the job (hence the plugin) @param jobId: the Job identification number @type jobId: string @return: the "EDJob instance", which contains the plugin (__edPlugin) and the status @rtype: a Python object. """ if jobId in cls.__dictJobs: return cls.__dictJobs[jobId] else: EDVerbose.WARNING("Unable to retrieve such EDJob: %s" % jobId) getJobFromId = getJobFromID @classmethod def getMemoryFootprint(cls): if asizeof is not None: return asizeof.asizesof(cls.__dictJobs) def cleanJob(self, forceGC=True): """ Frees the memory associated with the top level plugin @param forceGC: Force garbage collection after clean-up @type forceGC: boolean """ self.synchronize() with self.locked(): if self.__edPlugin is not None: self.__pathXSDOutput = self.__edPlugin.strPathDataOutput self.__pathXSDInput = self.__edPlugin.strPathDataInput self.__runtime = self.__edPlugin.getRunTime() self.__edPlugin = None if forceGC: gc.collect() @classmethod def cleanJobfromId(cls, jobId, forceGC=True): """ Frees the memory associated with the top level plugin @param jobId: the Job identification number @type jobId: string @param forceGC: Force garbage collection after clean-up @type forceGC: boolean """ if jobId in cls.__dictJobs: job = cls.__dictJobs[jobId] job.cleanJob(forceGC) strRet = "Job %s cleaned" % jobId else: strRet = "Unable to retrieve such EDJob: %s" % jobId EDVerbose.WARNING(strRet) return strRet cleanJobfromID = cleanJobfromId @classmethod def getDataOutputFromId(cls, jobId): """ Returns the Plugin Output Data @param jobId: job idenfier @type jobId: string @return: edJob.DataOutput XML string """ output = None job = cls.getJobFromId(jobId) if job is not None: output = job.getDataOutput() return output or "" getDataOutputFromID = getDataOutputFromId @classmethod def getDataInputFromId(cls, jobId): """ Returns the Plugin Input Data @param jobId: job idenfier @type jobId: string @return: edJob.DataInput XML string """ output = None job = cls.getJobFromId(jobId) if job is not None: output = job.getDataInput() return output or "" getDataInputFromID = getDataInputFromId @classmethod def countRunning(cls): """ return the number of jobs still running. """ running = 0 for jobid in list(cls.__dictJobs.keys()): # python3 ! running += ( cls.__dictJobs[jobid].__status == cls.PLUGIN_STATE_RUNNING) return running @classmethod def stats(cls): """ Retrieve some statistics and print them """ lstStrOut = [] output = [] fExecTime = time.time() - cls.__fStartTime keys = cls.__dictJobs.keys() keys.sort() for num, key in enumerate(keys): job = cls.__dictJobs[key] if job.getPlugin() is None: runtime = job.__runtime else: runtime = job.getPlugin().getRunTime() output.append( [num, key, job.getStatus(), runtime, job.getMemSize()]) output.sort() iNbJob = max(1, len(keys)) lstStrOut.append("_" * 110) lstStrOut.append( "%s\t|\t%s\t\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("nr", "EDPluginName-Id", "status", "runtime", "memory")) lstStrOut.append("_" * 110) fWall = 0.0 fSumProd = 0.0 fSumX = 0.0 fSumX2 = 0.0 for oneJob in output: fWall += oneJob[3] fSumX += oneJob[0] fSumX2 += oneJob[0] * oneJob[0] fSumProd += oneJob[0] * oneJob[3] lstStrOut.append("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob)) lstStrOut.append("_" * 110) lstStrOut.append( "Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime)) lstStrOut.append( "Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob)) if len(keys) > 1: fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX) fOrd = (fWall - fSlope * fSumX) / iNbJob else: fSlope = 0.0 fOrd = fWall lstStrOut.append( "Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope)) strOutput = os.linesep.join(lstStrOut) EDVerbose.screen(strOutput) return strOutput
class EDApplication(object): """ This is the main EDNA application class. This class can be sub-classed for any specific application need. An EDNA application is able to launch an entry point plugin. It accepts the following parameter: --execute : name of the plugin to be executed --inputFile : related plugin data (xml input data file name) --outputFile : related plugin result (xml output data file name) --conf : configuration file name --basedir : where the application working directory should go --DEBUG or --debug : turns on debugging -v or --version : Displays the application name and version --verbose : Turns on verbose mode --no-log : Turns off logging -h or --help : Prints out an usage message """ CONFIGURATION_PARAM_LABEL = "--conf" PLUGIN_PARAM_LABEL = "--execute" DATASET_PARAM_LABEL = "--inputFile" OUTPUT_PARAM_LABEL = "--outputFile" DATASET_BASE_DIRECTORY = "--basedir" DEBUG_PARAM_LABEL_1 = "--DEBUG" DEBUG_PARAM_LABEL_2 = "--debug" VERSION_PARAM_LABEL_1 = "-v" VERSION_PARAM_LABEL_2 = "--version" VERBOSE_MODE_LABEL = "--verbose" NO_LOG_LABEL = "--no-log" HELP_LABEL_1 = "-h" HELP_LABEL_2 = "--help" __edConfiguration = None __edFactoryPlugin = None __semaphore = Semaphore() def __init__(self, _strName="EDApplication", \ _strVersion="1.0.1", \ _strPluginName=None, \ _strConfigurationFileName=None, \ _strDataInputFilePath=None, \ _edLogFile=None, \ _strBaseDir=None, \ _strWorkingDir=None, \ _strDataOutputFilePath=None): self.__strName = _strName self.__strVersion = _strVersion self.__strPluginName = _strPluginName self.__strConfigurationFileName = _strConfigurationFileName self.__strDataInputFilePath = _strDataInputFilePath self.__strDataOutputFilePath = _strDataOutputFilePath self.__edLogFile = _edLogFile self.__strBaseDir = _strBaseDir self.__strWorkingDir = _strWorkingDir self.__strFullApplicationWorkingDirectory = None self.__strXMLData = None self.__listErrorMessages = [] self.__listWarningMessages = [] self.__xsDataOutput = None self.__edObtainedOutputDataFile = None self.__strDataOutputFilePath = None self.__edPlugin = None self.__edCommandLine = EDCommandLine(sys.argv) self.__strApplicationInstanceName = self.__strName + "_" + time.strftime("%Y%m%d-%H%M%S", time.localtime(time.time())) self.__strLogFileName = self.__strApplicationInstanceName + ".log" self.__bIsFailure = False self.__strCurrentWorkingDirectory = os.getcwd() self.__strConfigurationHome = None self.__strPathToLogFile = None def execute(self): """ This is the main execute method which executes preProcess, process and postProcess. """ self.preProcess() self.process() self.postProcess() def preProcess(self): """ Creates the application working directory (log dir) Initializes the configuration retrieves the plugin xml data to be passed to the plugin """ EDVerbose.DEBUG("EDApplication.preProcess") self.processCommandline() if (not self.__bIsFailure): # Check that the plugin can be located strPluginLocation = EDFactoryPluginStatic.getFactoryPlugin().getModuleLocation(self.__strPluginName) if (strPluginLocation is None): EDVerbose.error("Plugin %s cannot be loaded!" % self.__strPluginName) self.__bIsFailure = True # Check that the input file can be read if (self.getDataInputFilePath() is not None) and (not os.path.exists(self.__strDataInputFilePath)): EDVerbose.error("Input XML file not found : %s" % self.__strDataInputFilePath) self.__bIsFailure = True # Check that the output file can be created if (self.__strDataOutputFilePath is not None): strOutputDirectory = os.path.dirname(self.__strDataOutputFilePath) if (strOutputDirectory is None or strOutputDirectory == ""): strOutputDirectory = os.getcwd() self.__strDataOutputFilePath = os.path.join(strOutputDirectory, self.__strDataOutputFilePath) if (not os.access(strOutputDirectory, os.W_OK)): EDVerbose.error("Output directory not writable: %s" % strOutputDirectory) self.__bIsFailure = True elif (os.path.exists(self.__strDataOutputFilePath)): if (not os.access(self.__strDataOutputFilePath, os.W_OK)): EDVerbose.error("Output file not writable: %s" % self.__strDataOutputFilePath) self.__bIsFailure = True if (not self.__bIsFailure): EDVerbose.DEBUG("EDApplication.PLUGIN_PARAM_LABEL: " + EDApplication.PLUGIN_PARAM_LABEL) # Load the configuration file if (os.path.exists(self.__strConfigurationFileName)): EDVerbose.screen("Loading Configuration file: %s" % self.__strConfigurationFileName) EDConfigurationStatic.addConfigurationFile(self.__strConfigurationFileName, _bReplace=True) pyDictionary = {} pyDictionary[ "${EDNA_HOME}" ] = EDUtilsPath.getEdnaHome() if self.getDataInputFilePath() is not None: self.__strXMLData = EDUtilsFile.readFileAndParseVariables(self.getDataInputFilePath(), pyDictionary) else: EDVerbose.warning("Cannot find configuration file: %s" % self.__strConfigurationFileName) # Create the application working directory if(self.__strWorkingDir is None): self.__strWorkingDir = self.__strApplicationInstanceName self.createApplicationWorkingDirectory() def process(self): """ Calls the Plugin to be executed """ if (not self.__bIsFailure): self.__edPlugin = EDFactoryPluginStatic.loadPlugin(self.__strPluginName) if(self.__edPlugin is not None): self.__edPlugin.setBaseDirectory(self.__strFullApplicationWorkingDirectory) self.__edPlugin.setBaseName(self.__strPluginName) self.__edPlugin.setDataInput(self.__strXMLData) self.__edPlugin.connectSUCCESS(self.doSuccessActionPlugin) self.__edPlugin.connectFAILURE(self.doFailureActionPlugin) EDVerbose.DEBUG("EDApplication.process: Executing " + self.__strPluginName) self.__edPlugin.execute() self.__edPlugin.synchronize() else: EDVerbose.error(EDMessage.ERROR_PLUGIN_NOT_LOADED_02 % ('EDApplication.process', self.__strPluginName)) self.__bIsFailure = True def processCommandline(self): """ This method is intended to be overridden by applications who would like to implement their own command line handling. This default method implements the following workflow: - Check for debug, verbose and log file command line options """ EDVerbose.DEBUG("EDApplication.execute") EDVerbose.log(self.__edCommandLine.getCommandLine()) self.processCommandLineDebugVerboseLogFile() # Determine the base directory if(self.__strBaseDir is None): self.processCommandLineBaseDirectory() # Set the name of the log file self.__strPathToLogFile = os.path.abspath(os.path.join(self.__strBaseDir, self.__strLogFileName)) EDVerbose.setLogFileName(self.__strPathToLogFile) self.processCommandLineHelp() if (not self.__bIsFailure): self.processCommandLineVersion() if (not self.__bIsFailure): # Name of the plugin to be executed if (self.__strPluginName is None): self.processCommandLinePluginName() # Path to the input XML file if (self.__strDataInputFilePath is None): self.processCommandLineInputFilePath() # Path to the output XML file if(self.__strDataOutputFilePath is None): self.processCommandLineOutputFilePath() if (self.__bIsFailure): self.usage() if (not self.__bIsFailure): # If strConfigurationFileName is None, this means that it has not been given to the constructor\ # It has been given by the command line\ if(self.__strConfigurationFileName is None): self.__strConfigurationFileName = self.getCommandLineArgument(EDApplication.CONFIGURATION_PARAM_LABEL) def processCommandLineDebugVerboseLogFile(self): EDVerbose.DEBUG("EDApplication.processCommandLineDebugVerboseLogFile") EDVerbose.setVerboseOff() # Check if no log file if (self.__edCommandLine.existCommand(EDApplication.NO_LOG_LABEL)): EDVerbose.setLogFileOff() EDVerbose.DEBUG("Log file output switched off") # Check if debug mode if (self.__edCommandLine.existCommand(EDApplication.DEBUG_PARAM_LABEL_1) or self.__edCommandLine.existCommand(EDApplication.DEBUG_PARAM_LABEL_2)): EDVerbose.setVerboseDebugOn() EDVerbose.DEBUG("Debug Mode [ON]") # Check if verbose if (self.__edCommandLine.existCommand(EDApplication.VERBOSE_MODE_LABEL)): EDVerbose.setVerboseOn() def processCommandLineHelp(self): EDVerbose.DEBUG("EDApplication.processCommandLineHelp") if (self.__edCommandLine.existCommand(EDApplication.HELP_LABEL_1) or self.__edCommandLine.existCommand(EDApplication.HELP_LABEL_2)): EDVerbose.setVerboseOn() self.usage() self.__bIsFailure = True def processCommandLineVersion(self): EDVerbose.DEBUG("EDApplication.processCommandLineVersion") if (self.__edCommandLine.existCommand(EDApplication.VERSION_PARAM_LABEL_1) or self.__edCommandLine.existCommand(EDApplication.VERSION_PARAM_LABEL_2)): EDVerbose.setVerboseOn() EDVerbose.screen("%s version %s" % (self.__strName, self.__strVersion)) self.__bIsFailure = True def processCommandLinePluginName(self): """ """ EDVerbose.DEBUG("EDApplication.processCommandLinePluginName") if (not self.__edCommandLine.existCommand(EDApplication.PLUGIN_PARAM_LABEL)): EDVerbose.error("No %s command line argument found!" % EDApplication.PLUGIN_PARAM_LABEL) self.__bIsFailure = True else: self.__strPluginName = self.getCommandLineArgument(EDApplication.PLUGIN_PARAM_LABEL) EDVerbose.DEBUG("EDApplication.processCommandLinePluginName : %s = %s" % (EDApplication.PLUGIN_PARAM_LABEL, self.__strPluginName)) def processCommandLineInputFilePath(self): """ """ EDVerbose.DEBUG("EDApplication.processCommandLineInputFilePath") if (not self.__edCommandLine.existCommand(EDApplication.DATASET_PARAM_LABEL)): EDVerbose.error("No %s command line argument found!" % EDApplication.DATASET_PARAM_LABEL) self.__bIsFailure = True else: self.__strDataInputFilePath = self.getCommandLineArgument(EDApplication.DATASET_PARAM_LABEL) EDVerbose.DEBUG("EDApplication.initApplication : %s = %s" % (EDApplication.DATASET_PARAM_LABEL, self.__strDataInputFilePath)) def processCommandLineOutputFilePath(self): """ """ EDVerbose.DEBUG("EDApplication.processCommandLineOutputFilePath") if (not self.__edCommandLine.existCommand(EDApplication.OUTPUT_PARAM_LABEL)): EDVerbose.DEBUG("No %s command line argument found" % EDApplication.OUTPUT_PARAM_LABEL) else: self.__strDataOutputFilePath = self.getCommandLineArgument(EDApplication.OUTPUT_PARAM_LABEL) EDVerbose.DEBUG("EDApplication.initApplication : %s = %s" % (EDApplication.OUTPUT_PARAM_LABEL, self.__strDataOutputFilePath)) def processCommandLineBaseDirectory(self): """ """ EDVerbose.DEBUG("EDApplication.processCommandLineBaseDirectory") self.__strBaseDir = self.getCommandLineArgument(EDApplication.DATASET_BASE_DIRECTORY) if(self.__strBaseDir is None): self.__strBaseDir = os.getcwd() EDVerbose.DEBUG("Base directory set to current working directory = %s" % (self.__strBaseDir)) else: EDVerbose.DEBUG("%s = %s" % (EDApplication.DATASET_BASE_DIRECTORY, self.__strBaseDir)) def postProcess(self): """ """ # Restore the current working directory os.chdir(self.__strCurrentWorkingDirectory) @classmethod def usage(cls): """ Print usage... """ EDVerbose.screen("") EDVerbose.screen("Usage: ") EDVerbose.screen("") EDVerbose.screen("%35s : Name of the plugin to be executed" % (cls.PLUGIN_PARAM_LABEL)) EDVerbose.screen("") EDVerbose.screen("%35s : Path to the XML input file" % (cls.DATASET_PARAM_LABEL)) EDVerbose.screen("") EDVerbose.screen("-----------------------------------------------------------------------------------------------------------") EDVerbose.screen("") EDVerbose.screen(" Additional options available:") EDVerbose.screen("") EDVerbose.screen("%35s : Path to the file wich will contain the XML output" % (cls.OUTPUT_PARAM_LABEL)) EDVerbose.screen("") EDVerbose.screen("%35s : Base directory, i.e. working directory for the application" % (cls.DATASET_BASE_DIRECTORY)) EDVerbose.screen("") EDVerbose.screen("%35s : Verbose mode" % (cls.VERBOSE_MODE_LABEL)) EDVerbose.screen("") EDVerbose.screen("%35s : XSConfiguration file" % (cls.CONFIGURATION_PARAM_LABEL)) EDVerbose.screen("") EDVerbose.screen("%35s : Executable version info" % (cls.VERSION_PARAM_LABEL_1 + " or " + cls.VERSION_PARAM_LABEL_2)) EDVerbose.screen("") EDVerbose.screen("%35s : DEBUG log traces" % (cls.DEBUG_PARAM_LABEL_1 + " or " + cls.DEBUG_PARAM_LABEL_2)) EDVerbose.screen("") EDVerbose.screen("%35s : No log file" % (cls.NO_LOG_LABEL)) EDVerbose.screen("") EDVerbose.screen("%35s : This help message" % (cls.HELP_LABEL_1 + " or " + cls.HELP_LABEL_2)) EDVerbose.screen("") @classmethod def getFactoryPlugin(cls): EDVerbose.WARNING("the use of EDclsetFactoryPlugin is deprecated. Please use EDFactoryPluginStatic.getFactoryPlugin instead.") return EDFactoryPluginStatic.getFactoryPlugin @classmethod def loadPlugin(cls, _strPluginName): EDVerbose.WARNING("The use of EDApplication.loadPlugin is deprecated. Please use EDFactoryPluginStatic.getFactoryPlugin instead.") return EDFactoryPluginStatic.loadPlugin(_strPluginName) @classmethod def loadModule(cls, _strModuleName): EDVerbose.WARNING("The use of EDApplication.loadModule is deprecated. Please use EDFactoryPluginStatic.getFactoryPlugin instead.") EDFactoryPluginStatic.loadModule(_strModuleName) def getDataInputFilePath(self): return self.__strDataInputFilePath def getBaseDir(self): """ Getter for base directory @return: path of the base directory @rtype: string """ return self.__strBaseDir def createApplicationWorkingDirectory(self): """ Created the working directory of the application (<date>-<application name>) First tries to retrieve the base dir from --basedir option or related parameter from constructor Otherwise tries to retrieve it from EDNA_BASE_DIRECTORY environment variable Otherwise put the base dir as the current directory """ EDVerbose.DEBUG("EDApplication.createApplicationWorkingDirectory") strBaseDirectory = self.getBaseDir() strDateTime = time.strftime("%Y%m%d-%H%M%S", time.localtime(time.time())) self.__strFullApplicationWorkingDirectory = os.path.join(strBaseDirectory, self.__strWorkingDir) # Check that a folder / file with the same name already exists if(os.path.exists(self.__strFullApplicationWorkingDirectory) or \ os.path.exists(self.__strFullApplicationWorkingDirectory)): # It does exist so we have to modify the name of the working directory iIndex = 1 bContinueFlag = True while (bContinueFlag): self.__strFullApplicationWorkingDirectory = os.path.join(strBaseDirectory, "%s_%d" % \ (strDateTime, \ iIndex)) if(os.path.isdir(self.__strFullApplicationWorkingDirectory) or \ os.path.exists(self.__strFullApplicationWorkingDirectory)): iIndex += 1 else: bContinueFlag = False # Make the directory os.mkdir(self.__strFullApplicationWorkingDirectory) # Change it to be the current working directory os.chdir(self.__strFullApplicationWorkingDirectory) def getFullApplicationWorkingDirectory(self): return self.__strFullApplicationWorkingDirectory def getCurrentWorkingDirectory(self): return self.__strCurrentWorkingDirectory def doSuccessActionPlugin(self, _edPlugin): """ """ EDVerbose.DEBUG("EDApplication.doSuccessActionPlugin") # Print the potential Warnings and Errors self.__listWarningMessages = _edPlugin.getListOfWarningMessages() EDVerbose.DEBUG("EDApplication.doSuccessActionPlugin: Plugin %s Successful with : %i Warnings " % (_edPlugin.getPluginName(), len(self.__listWarningMessages))) for warningMessage in self.__listWarningMessages: EDVerbose.screen(warningMessage) self.__listErrorMessages = _edPlugin.getListOfErrorMessages() EDVerbose.DEBUG("EDApplication.doSuccessActionPlugin: Plugin %s Successful with : %i Errors" % (_edPlugin.getPluginName(), len(self.__listErrorMessages))) for errorMessage in self.__listErrorMessages: EDVerbose.error(errorMessage) if (_edPlugin.hasDataOutput()): xsDataOutput = _edPlugin.getDataOutput() if (xsDataOutput is not None and self.__strDataOutputFilePath is not None): xsDataOutput.exportToFile(self.__strDataOutputFilePath) if (xsDataOutput is not None and self.__edObtainedOutputDataFile is not None): xsDataOutput.exportToFile(self.__edObtainedOutputDataFile) def doFailureActionPlugin(self, _edPlugin): EDVerbose.DEBUG("EDApplication.doFailureActionPlugin") # Print the potential Warnings and Errors EDVerbose.DEBUG("EDApplication.doFailureActionPlugin: Plugin %s failed" % _edPlugin.getClassName()) self.__listWarningMessages = _edPlugin.getListOfWarningMessages() for warningMessage in self.__listWarningMessages: EDVerbose.screen(warningMessage) self.__listErrorMessages = _edPlugin.getListOfErrorMessages() for errorMessage in self.__listErrorMessages: EDVerbose.screen(errorMessage) if (_edPlugin.hasDataOutput()): xsDataOutput = _edPlugin.getDataOutput() if (xsDataOutput is not None and self.__strDataOutputFilePath is not None): xsDataOutput.exportToFile(self.__strDataOutputFilePath) if (xsDataOutput is not None and self.__edObtainedOutputDataFile is not None): xsDataOutput.exportToFile(self.__edObtainedOutputDataFile) def getPlugin(self): return self.__edPlugin def getPluginOutputData(self): return self.__xsDataOutput def getWarningMessages(self): return self.__listWarningMessages def getErrorMessages(self): return self.__listErrorMessages def getEdCommandLine(self): return self.__edCommandLine def getCommandLine(self): return self.__edCommandLine.getCommandLine() def getCommandLineArguments(self): with self.__class__.__semaphore: edCommandLine = self.__edCommandLine.getCommandLine() return edCommandLine def getCommandLineArgument(self, _strKey): with self.__class__.__semaphore: strCommandLineArgument = self.__edCommandLine.getArgument(_strKey) return strCommandLineArgument @classmethod def synchronizeOn(cls): """ Lock the whole class """ cls.__semaphore.acquire() @classmethod def synchronizeOff(cls): """ Unlock the whole class """ cls.__semaphore.release() def getApplicationName(self): return self.__strName + "-" + self.__strVersion def getWorkingDir(self): """ Getter for working dir @rtype: string @return working dir """ return self.__strWorkingDir def setWorkingDir(self, _strDir): """ Setter for working dir @type _strDir: string @param _strDir: working dir """ self.__strWorkingDir = _strDir def isFailure(self): return self.__bIsFailure def setFailure(self, _bFailure): self.__bIsFailure = _bFailure def getPluginName(self): return self.__strPluginName
class EDPluginBioSaxsSmartMergev1_7(EDPluginControl): """ This plugin takes a set of input data files (1D SAXS) measure their differences (versus previous and versus first) and merge those which are equivalent v1.1 adds information from AutoSub Controlled plugins: - Execplugins/WaitMultifile - EdnaSaxs/Atsas/DatAver - EdnaSaxs/Atsas/AutoSub - EdnaSaxs/SaxsAnalysis """ # dictAve = {} #key=?; value=path to average file lastBuffer = None lastSample = None dictFrames = { } #dictionary key = filename of average, value = list of files used for average __strControlledPluginDataver = "EDPluginExecDataverv2_0" #__strControlledPluginDatcmp = "EDPluginExecDatcmpv2_0" __strControlledPluginWaitFile = "EDPluginWaitMultiFile" __strControlledPluginAutoSub = "EDPluginAutoSubv1_1" __strControlledPluginSaxsAnalysis = "EDPluginControlSaxsAnalysisv1_0" __strControlledPluginSaxsISPyB = "EDPluginBioSaxsISPyBv1_0" __configured = False CONF_MINIMUM_CURVE_FILE_SIZE = "MinCurveFileSize" minimumCurveFileSize = 10000 semaphore = Semaphore() cpRsync = "EDPluginExecRsync" def __init__(self): """ """ EDPluginControl.__init__(self) #self.__edPluginExecDatcmp = None self.__edPluginExecDataver = None self.__edPluginExecWaitFile = None self.__edPluginExecAutoSub = None self.__edPluginSaxsAnalysis = None self.__edPluginSaxsISPyB = None self.setXSDataInputClass(XSDataInputBioSaxsSmartMergev1_0) #self.__edPluginExecDatCmp = None self.lstInput = [] self.curves = [] self.forgetLastSample = False self.lstMerged = [] self.lstDiscarded = [] self.lstXsdInput = [] self.absoluteFidelity = 0.01 #None self.relativeFidelity = 0.01 #None self.dictSimilarities = {} # key: 2-tuple of images, similarities self.lstStrInput = [] self.autoRg = None self.gnom = None self.volume = None self.rti = None self.strRadiationDamage = None self.strMergedFile = None self.lstSub = [] self.strSubFile = None self.fConcentration = None self.xsDataResult = XSDataResultBioSaxsSmartMergev1_0() self.xsBestBuffer = None self.bestBufferType = "" self.bufferFrames = [] self.xsScatterPlot = None self.xsGuinierPlot = None self.xsKratkyPlot = None self.xsDensityPlot = None self.xsdSubtractedCurve = None self.outdir = None #directory on rnice for analysis results to go to def configure(self): """ Configures the plugin from the configuration file with the following parameters: - curve_file_size: minimum size of the file. """ EDPluginControl.configure(self) if not self.__configured: with self.semaphore: if not self.__configured: self.DEBUG("EDPluginBioSaxsSmartMergev1_7.configure") min_size = self.config.get( self.CONF_MINIMUM_CURVE_FILE_SIZE) if min_size is None: strMessage = 'EDPluginBioSaxsSmartMergev1_7.configure: %s Configuration parameter missing: \ %s, defaulting to "%s"' % (self.getBaseName(), self.CONF_MINIMUM_CURVE_FILE_SIZE, self.minimumCurveFileSize) self.WARNING(strMessage) self.addErrorWarningMessagesToExecutiveSummary( strMessage) else: self.__class__.minimumCurveFileSize = float(min_size) self.__class__.__configured = True def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginBioSaxsSmartMergev1_7.checkParameters") self.checkMandatoryParameters(self.dataInput, "Data Input is None") self.checkMandatoryParameters(self.dataInput.inputCurves, "Input curve list is empty") self.checkMandatoryParameters(self.dataInput.mergedCurve, "Output curve filename is empty") def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_7.preProcess") # Load the execution plugin if self.dataInput.absoluteFidelity is not None: self.absoluteFidelity = self.dataInput.absoluteFidelity.value if self.absoluteFidelity == 0.0: self.absoluteFidelity = None if self.dataInput.relativeFidelity is not None: self.relativeFidelity = self.dataInput.relativeFidelity.value if self.relativeFidelity == 0.0: self.relativeFidelity = None self.lstInput = self.dataInput.inputCurves self.lstStrInput = [i.path.value for i in self.lstInput] self.__edPluginExecWaitFile = self.loadPlugin( self.__strControlledPluginWaitFile) if self.dataInput.mergedCurve is not None: self.strMergedFile = self.dataInput.mergedCurve.path.value if self.dataInput.subtractedCurve is not None: self.strSubFile = self.dataInput.subtractedCurve.path.value dirname = os.path.dirname(self.strSubFile) if not os.path.isdir(dirname): os.mkdir(dirname) def getFramesByFilename(self, filename): if filename is not None: if filename in self.__class__.dictFrames.keys(): return self.__class__.dictFrames[filename] return {"averaged": [], "discarded": []} def getAveragedFrameByFilename(self, filename): frames = self.getFramesByFilename(filename) if frames is not None: return frames["averaged"] return [] def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_7.process") xsdwf = XSDataInputWaitMultiFile( timeOut=XSDataTime(30), expectedSize=XSDataInteger(self.minimumCurveFileSize), expectedFile=[XSDataFile(i.path) for i in self.lstInput]) self.__edPluginExecWaitFile.setDataInput(xsdwf) self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait) self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait) self.__edPluginExecWaitFile.executeSynchronous() if self.isFailure(): return if len(self.lstInput) == 1: inp = self.lstInput[0].path.value dst = self.dataInput.mergedCurve.path.value if not os.path.isdir(os.path.dirname(dst)): self.error("Output directory for %s does not exist" % dst) os.makedirs(os.path.dirname(dst)) if not os.path.exists(inp): self.warning("Input %s does not (yet?) exist" % inp) time.sleep(1.0) shutil.copyfile(inp, dst) self.addExecutiveSummaryLine( "Got only one frame ... nothing to merge !!!") else: self.lstMerged = [] if (self.absoluteFidelity is not None) or (self.relativeFidelity is not None): if self.absoluteFidelity is not None: for idx, oneFile in enumerate(self.lstInput[1:]): self.DEBUG("Calculating similarity of 0 and %s" % idx) self.datcmp(self.lstInput[0], oneFile) # edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) # xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], oneFile]) # edPluginExecAbsoluteFidelity.setDataInput(xsd) # edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp) # edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp) # edPluginExecAbsoluteFidelity.execute() if (self.relativeFidelity is not None): if (self.absoluteFidelity is None): self.DEBUG("Calculating similarity of 0 and 1") self.datcmp(self.lstInput[0], self.lstInput[1]) # edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) # xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], self.lstInput[1] ]) # edPluginExecAbsoluteFidelity.setDataInput(xsd) # edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp) # edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp) # edPluginExecAbsoluteFidelity.execute() if (len(self.lstInput) > 2): for idx, oneFile in enumerate(self.lstInput[2:]): self.DEBUG("Calculating similarity of %s and %s" % (idx, idx + 1)) self.datcmp(self.lstInput[idx + 1], oneFile) # edPluginExecRelativeFidelity = self.loadPlugin(self.__strControlledPluginDatcmp) # xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[idx + 1], oneFile]) # edPluginExecRelativeFidelity.setDataInput(xsd) # edPluginExecRelativeFidelity.connectFAILURE(self.doFailureExecDatcmp) # edPluginExecRelativeFidelity.connectSUCCESS(self.doSuccessExecDatcmp) # edPluginExecRelativeFidelity.execute() self.synchronizePlugins() for idx, oneFile in enumerate(self.lstInput): if idx == 0: self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None) and (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity) and (self.dictSimilarities[ (idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) elif (self.absoluteFidelity is not None): if (0, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (0, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(0, idx)] >= self.absoluteFidelity): self.lstMerged.append(oneFile) elif (self.relativeFidelity is not None): if (idx - 1, idx) not in self.dictSimilarities: self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) + "\n".join([ "%s: %s" % (key, self.dictSimilarities[key]) for key in self.dictSimilarities ])) self.resynchronize() if (self.dictSimilarities[(idx - 1, idx)] >= self.relativeFidelity): self.lstMerged.append(oneFile) else: self.lstMerged.append(oneFile) self.lstMerged.sort(cmp) if len(self.lstMerged) != len(self.lstInput): self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len( self.lstMerged) self.WARNING(self.strRadiationDamage) self.addExecutiveSummaryLine("WARNING: " + self.strRadiationDamage) self.addExecutiveSummaryLine("Merging files: " + " ".join( [os.path.basename(i.path.value) for i in self.lstMerged])) if len(self.lstMerged) == 1: self.rewriteHeader(self.lstMerged[0].path.value, self.strMergedFile) else: self.__edPluginExecDataver = self.loadPlugin( self.__strControlledPluginDataver) xsd = XSDataInputDataver(inputCurve=self.lstMerged) # outputCurve=self.dataInput.mergedCurve, self.__edPluginExecDataver.setDataInput(xsd) self.__edPluginExecDataver.connectSUCCESS( self.doSuccessExecDataver) self.__edPluginExecDataver.connectFAILURE( self.doFailureExecDataver) self.__edPluginExecDataver.executeSynchronous() if (self.fConcentration == 0) and (self.strSubFile is not None): if (self.__class__.lastBuffer is not None) and (self.__class__.lastSample is not None): self.__edPluginExecAutoSub = self.loadPlugin( self.__strControlledPluginAutoSub) base = "_".join( os.path.basename( self.__class__.lastSample.path.value).split("_")[:-1]) suff = os.path.basename(self.strSubFile).split("_")[-1] sub = os.path.join(os.path.dirname(self.strSubFile), base + "_" + suff) self.xsdSubtractedCurve = XSDataFile(XSDataString(sub)) #self.curves.append(xsdSubtractedCurve) self.__edPluginExecAutoSub.dataInput = XSDataInputAutoSub( sampleCurve=self.__class__.lastSample, buffers=[ self.__class__.lastBuffer, self.dataInput.mergedCurve ], subtractedCurve=self.xsdSubtractedCurve) self.__edPluginExecAutoSub.connectSUCCESS( self.doSuccessExecAutoSub) self.__edPluginExecAutoSub.connectFAILURE( self.doFailureExecAutoSub) self.__edPluginExecAutoSub.executeSynchronous() if self.isFailure(): return self.__edPluginSaxsAnalysis = self.loadPlugin( self.__strControlledPluginSaxsAnalysis) self.__edPluginSaxsAnalysis.dataInput = XSDataInputSaxsAnalysis( scatterCurve=self.xsdSubtractedCurve, autoRg=self.autoRg, graphFormat=XSDataString("png")) self.__edPluginSaxsAnalysis.connectSUCCESS( self.doSuccessSaxsAnalysis) self.__edPluginSaxsAnalysis.connectFAILURE( self.doFailureSaxsAnalysis) self.__edPluginSaxsAnalysis.executeSynchronous() self.__class__.lastBuffer = self.dataInput.mergedCurve #self.__class__.lastSample = None #Information neededfor transfer to ISPyB self.forgetLastSample = True else: self.__class__.lastSample = self.dataInput.mergedCurve if self.dataInput.sample and self.dataInput.sample.login and self.dataInput.sample.passwd and self.dataInput.sample.measurementID: self.addExecutiveSummaryLine("Registering to ISPyB") self.lstDiscarded = list(set(self.lstInput) - set(self.lstMerged)) self.__class__.dictFrames[self.dataInput.mergedCurve] = { 'averaged': self.lstMerged, 'discarded': self.lstDiscarded } self.__edPluginSaxsISPyB = self.loadPlugin( self.__strControlledPluginSaxsISPyB) if len(self.lstInput) > 1: frameAverage = XSDataInteger(len(self.lstInput)) frameMerged = XSDataInteger(len(self.lstMerged)) else: frameMerged = frameAverage = XSDataInteger(1) self.curves = [XSDataFile(i.path) for i in self.lstInput] self.discardedCurves = [ XSDataFile(i.path) for i in self.lstDiscarded ] self.mergedCurves = [XSDataFile(i.path) for i in self.lstMerged] averageFilePath = None if self.strMergedFile is not None: averageFilePath = XSDataFile(XSDataString(self.strMergedFile)) self.sampleFrames = self.getAveragedFrameByFilename( self.__class__.lastSample) lastSample = None if self.__class__.lastSample is not None: lastSample = self.__class__.lastSample subtractedCurve = None if self.xsdSubtractedCurve is not None: subtractedCurve = self.xsdSubtractedCurve else: subtractedCurve = None xsdin = XSDataInputBioSaxsISPyBv1_0( sample=self.dataInput.sample, autoRg=self.autoRg, gnom=self.gnom, volume=self.volume, frameAverage=frameAverage, frameMerged=frameMerged, curves=self.curves, discardedFrames=self.discardedCurves, averagedFrames=self.mergedCurves, averageFilePath=averageFilePath, bufferFrames=self.bufferFrames, sampleFrames=self.sampleFrames, bestBuffer=self.xsBestBuffer, averageSample=lastSample, scatterPlot=self.xsScatterPlot, guinierPlot=self.xsGuinierPlot, kratkyPlot=self.xsKratkyPlot, densityPlot=self.xsDensityPlot, subtractedFilePath=subtractedCurve # destination=self.dataInput.sample.ispybDestination #duplicate, already in sample ) self.__edPluginSaxsISPyB.dataInput = xsdin self.__edPluginSaxsISPyB.connectSUCCESS(self.doSuccessISPyB) self.__edPluginSaxsISPyB.connectFAILURE(self.doFailureISPyB) self.__edPluginSaxsISPyB.execute() # transfer analysis data to correct location on nice if self.gnom is not None: self.outdir = os.path.join( os.path.dirname(os.path.dirname(self.lstStrInput[0])), "ednaAnalysis") basename = os.path.basename( os.path.splitext(self.gnom.gnomFile.path.value)[0]) self.outdir = os.path.join(self.outdir, basename) if not os.path.isdir(self.outdir): os.makedirs(self.outdir) #self.outFile = os.path.join(outdir, "NoResults.html") workingdir = os.path.dirname(self.gnom.gnomFile.path.value) self.pluginRsync = self.loadPlugin(self.cpRsync) self.pluginRsync.dataInput = XSDataInputRsync( source=XSDataFile(XSDataString(workingdir)), destination=XSDataFile(XSDataString(self.outdir)), options=XSDataString("-avx")) self.pluginRsync.connectSUCCESS(self.doSuccessExecRsync) self.pluginRsync.connectFAILURE(self.doFailureExecRsync) self.pluginRsync.executeSynchronous() if self.forgetLastSample: #Also redefine dictionary to contain the buffer just processed? self.__class__.lastSample = None def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginBioSaxsSmartMergev1_7.postProcess") # Create some output data self.xsDataResult.mergedCurve = self.dataInput.mergedCurve if self.strSubFile is not None and os.path.isfile(self.strSubFile): self.xsDataResult.subtractedCurve = XSDataFile( XSDataString(self.strSubFile)) self.xsDataResult.autoRg = self.autoRg self.xsDataResult.gnom = self.gnom self.xsDataResult.volume = self.volume self.xsDataResult.rti = self.rti def finallyProcess(self, _edObject=None): EDPluginControl.finallyProcess(self) self.xsDataResult.status = XSDataStatus( message=self.getXSDataMessage(), executiveSummary=XSDataString( os.linesep.join(self.getListExecutiveSummaryLines()))) self.dataOutput = self.xsDataResult def resynchronize(self): """ Sometimes plugins are not started or not yet finished... """ time.sleep(1) # this is just a way to give the focus to other threads self.synchronizePlugins() self.ERROR("I slept a second, waiting for sub-plugins to finish") def rewriteHeader(self, infile=None, output=None, hdr="#", linesep=os.linesep): """ Write the output file with merged data with the good header. # BSA sample #Sample c= 0.0 mg/ml # # Merged from: file1 # Merged from: file2 # Merged from: file4 # # Sample Information: # Concentration: 0.0 # Code: BSA @param infile: path of the original data file where original data are taken from @param outfile: path of the destination file (by default, from XSD) @param hdr: header marker, often "#" @param linesep: line separator, usually "\n" or "\r\n" depending on the Operating System @return: None """ Code = Concentration = None frameMax = exposureTime = measurementTemperature = storageTemperature = None originalFile = self.lstMerged[0].path.value headers = [ line.strip() for line in open(originalFile) if line.startswith("#") ] Comments = headers[0][1:].strip() for line in headers: if "title =" in line: Comments = line.split("=", 1)[1].strip() elif "Comments =" in line: Comments = line.split("=", 1)[1].strip() elif "Concentration:" in line: Concentration = line.split(":", 1)[1].strip() elif "Concentration =" in line: Concentration = line.split("=", 1)[1].strip() elif "Code =" in line: Code = line.split("=", 1)[1].strip() elif "Code:" in line: Code = line.split(":", 1)[1].strip() elif "Storage Temperature" in line: storageTemperature = line.split(":", 1)[1].strip() elif "Measurement Temperature" in line: measurementTemperature = line.split(":", 1)[1].strip() elif "Time per frame" in line: exposureTime = line.split("=", 1)[1].strip() elif "Frame" in line: frameMax = line.split()[-1] try: c = float(Concentration) except Exception: c = -1.0 self.fConcentration = c lstHeader = [ "%s %s" % (hdr, Comments), "%s Sample c= %s mg/ml" % (hdr, Concentration), hdr ] if frameMax is not None: lstHeader.append(hdr + " Number of frames collected: %s" % frameMax) if exposureTime is not None: lstHeader.append(hdr + " Exposure time per frame: %s" % exposureTime) if self.strRadiationDamage is None: lstHeader.append( "%s No significant radiation damage detected, merging %i files" % (hdr, len(self.lstMerged))) else: lstHeader.append("%s WARNING: %s" % (hdr, self.strRadiationDamage)) lstHeader += [ hdr + " Merged from: %s" % i.path.value for i in self.lstMerged ] if self.lstSub: lstHeader.append(hdr) lstHeader += ["%s %s" % (hdr, i) for i in self.lstSub] lstHeader += [hdr, hdr + " Sample Information:"] if storageTemperature is not None: lstHeader.append(hdr + " Storage Temperature (degrees C): %s" % storageTemperature) if measurementTemperature is not None: lstHeader.append(hdr + " Measurement Temperature (degrees C): %s" % measurementTemperature) lstHeader += [ hdr + " Concentration: %s" % Concentration, "# Code: %s" % Code ] if infile is None: infile = self.strMergedFile if output is None: output = self.strMergedFile data = linesep.join( [i.strip() for i in open(infile) if not i.startswith(hdr)]) with open(output, "w") as outfile: outfile.write(linesep.join(lstHeader)) if not data.startswith(linesep): outfile.write(linesep) outfile.write(data) def doSuccessExecWait(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecWait") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsSmartMergev1_6.doSuccessExecWait") self.retrieveMessages(_edPlugin) xsdo = _edPlugin.dataOutput self.DEBUG("ExecWait Output:%s" % xsdo.marshal()) if (xsdo.timedOut is not None) and bool(xsdo.timedOut.value): strErr = "Error in waiting for all input files to arrive" self.addExecutiveSummaryLine( "EDPluginBioSaxsSmartMergev1_6.doSuccessExecWait :" + strErr) self.ERROR(strErr) self.setFailure() def doFailureExecWait(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecWait") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureExecWait") self.retrieveMessages(_edPlugin) strErr = "Error in waiting for all input files to arrive" self.ERROR(strErr) self.addExecutiveSummaryLine(strErr) self.setFailure() def doSuccessExecDataver(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecDataver") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecDataver") self.retrieveMessages(_edPlugin) self.rewriteHeader(_edPlugin.dataOutput.outputCurve.path.value, output=self.strMergedFile) def doFailureExecDataver(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecDataver") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureExecDataver") self.retrieveMessages(_edPlugin) strErr = "Error in Processing of Atsas 'dataver'" self.addExecutiveSummaryLine(strErr) self.ERROR(strErr) self.setFailure() def datcmp(self, file1, file2): #def doSuccessExecDatcmp(self, _edPlugin=None): # self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecDatcmp") # self.retrieveSuccessMessages(_edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecDatcmp") # self.retrieveMessages(_edPlugin) data1 = loadtxt(file1.path.value) data2 = loadtxt(file2.path.value) with self.locked(): try: cor = gof(data1, data2) fidelity = cor.P except Exception as err: strErr = "No fidelity in output of datcmp" self.error(strErr) self.addExecutiveSummaryLine(strErr) self.addExecutiveSummaryLine(str(err)) #self.lstExecutiveSummary.append(strErr) #self.lstExecutiveSummary.append(err) if not "maximum recursion depth exceeded" in str(err): self.setFailure() fidelity = 0 lstIdx = [ self.lstStrInput.index(file1.path.value), self.lstStrInput.index(file2.path.value) ] lstIdx.sort() self.dictSimilarities[tuple(lstIdx)] = fidelity lstIdx.reverse() self.dictSimilarities[tuple(lstIdx)] = fidelity if fidelity <= 0: logFid = "infinity" else: logFid = "%.2f" % (-log(fidelity, 10)) self.addExecutiveSummaryLine( "-log(Fidelity) between %s and %s is %s" % (os.path.basename(file1.path.value), os.path.basename(file2.path.value), logFid)) # def doFailureExecDatcmp(self, _edPlugin=None): # self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecDatcmp") # self.retrieveFailureMessages(_edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureExecDatcmp") # self.retrieveMessages(_edPlugin) # self.addExecutiveSummaryLine("Failure in Processing of Atsas 'datcmp'") # self.setFailure() def doSuccessExecAutoSub(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecAutoSub") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecAutoSub") self.retrieveMessages(_edPlugin) self.autoRg = _edPlugin.dataOutput.autoRg if _edPlugin.dataOutput.subtractedCurve is not None: subcurve = _edPlugin.dataOutput.subtractedCurve if os.path.exists(subcurve.path.value): self.strSubFile = subcurve.path.value self.xsBestBuffer = _edPlugin.dataOutput.bestBuffer self.bestBufferType = _edPlugin.dataOutput.bestBufferType.value if self.bestBufferType == 'average': self.bufferFrames = self.lstMerged #if self.__class__.dictFrames[self.__class__.lastBuffer] is not None: # if self.__class__.dictFrames[self.__class__.lastBuffer]['averaged'] is not None: # self.bufferFrames = self.bufferFrames + self.__class__.dictFrames[self.__class__.lastBuffer]['averaged'] self.bufferFrames = self.bufferFrames + self.getAveragedFrameByFilename( self.__class__.lastBuffer) elif self.bestBufferType == self.__class__.lastBuffer: #if self.__class__.dictFrames[self.__class__.lastBuffer] is not None: # if self.__class__.dictFrames[self.__class__.lastBuffer]['averaged'] is not None: # self.bufferFrames = self.__class__.dictFrames[self.__class__.lastBuffer]['averaged'] self.bufferFrames = self.getAveragedFrameByFilename( self.__class__.lastBuffer) else: self.bufferFrames = self.lstMerged self.addExecutiveSummaryLine( _edPlugin.dataOutput.status.executiveSummary.value) def doFailureExecAutoSub(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecAutoSub") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doFailureExecAutoSub") self.retrieveMessages(_edPlugin) strErr = "Error in Processing of EDNA 'AutoSub'" # clean up "EDNA memory" self.__class__.lastBuffer = self.dataInput.mergedCurve self.__class__.lastSample = None self.ERROR(strErr) self.addExecutiveSummaryLine(strErr) self.setFailure() def doSuccessSaxsAnalysis(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessSaxsAnalysis") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessSaxsAnalysis") self.retrieveMessages(_edPlugin) self.gnom = _edPlugin.dataOutput.gnom self.volume = _edPlugin.dataOutput.volume self.rti = _edPlugin.dataOutput.rti self.xsScatterPlot = _edPlugin.dataOutput.scatterPlot self.xsGuinierPlot = _edPlugin.dataOutput.guinierPlot self.xsKratkyPlot = _edPlugin.dataOutput.kratkyPlot if hasattr(_edPlugin.dataOutput, 'kratkyRgPlot'): self.xsKratkyRgPlot = _edPlugin.dataOutput.kratkyRgPlot if hasattr(_edPlugin.dataOutput, 'kratkyVcPlot'): self.xsKratkyVcPlot = _edPlugin.dataOutput.kratkyVcPlot self.xsDensityPlot = _edPlugin.dataOutput.densityPlot self.addExecutiveSummaryLine( _edPlugin.dataOutput.status.executiveSummary.value) def doFailureSaxsAnalysis(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureSaxsAnalysis") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsSmartMergev1_6.doFailureSaxsAnalysis") self.retrieveMessages(_edPlugin) strErr = "Error in Processing of EDNA SaxsAnalysis = AutoRg => datGnom => datPorod" self.ERROR(strErr) self.addExecutiveSummaryLine(strErr) self.setFailure() def doSuccessISPyB(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessISPyB") self.addExecutiveSummaryLine("Registered in ISPyB") self.retrieveMessages(_edPlugin) def doFailureISPyB(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureISPyB") self.addExecutiveSummaryLine("Failed to registered in ISPyB") self.retrieveMessages(_edPlugin) def doSuccessExecRsync(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doSuccessExecRsync") self.retrieveSuccessMessages( _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doSuccessExecRsync") self.retrieveMessages(_edPlugin) self.gnom.gnomFile = XSDataFile( XSDataString( os.path.join(self.outdir, os.path.split(self.gnom.gnomFile.path.value)[1]))) def doFailureExecRsync(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsSmartMergev1_7.doFailureExecRsync") self.retrieveFailureMessages( _edPlugin, "EDPluginBioSaxsSmartMergev1_7.doFailureExecRsync") self.retrieveMessages(_edPlugin) self.setFailure()
class EDPluginBioSaxsProcessOneFilev1_4(EDPluginControl): """ Control plugin that does the same as previously without sub-plugin call ... except WaitFile which is still called. Nota normalization is done AFTER integration not before as previously New in version 1.4: return I/Q/stderr via XSDataArrays """ cpWaitFile = "EDPluginWaitFile" integrator = pyFAI.AzimuthalIntegrator() integrator.wavelength = 1e-10 CONF_DUMMY_PIXEL_VALUE = "DummyPixelValue" CONF_DUMMY_PIXEL_DELTA = "DummyPixelDelta" CONF_OPENCL_DEVICE = "DeviceType" __configured = False dummy = -2 delta_dummy = 1.1 semaphore = Semaphore() maskfile = None if pyFAI.opencl.ocl is None: METHOD = "lut" else: METHOD = "lut_ocl_gpu" def __init__(self): """ """ EDPluginControl.__init__(self) self.setXSDataInputClass(XSDataInputBioSaxsProcessOneFilev1_0) self.__edPluginWaitFile = None self.rawImage = None self.rawImageSize = XSDataInteger(1024) self.normalizedImage = None self.integratedCurve = None self.integratedImage = None self.lstExecutiveSummary = [] self.sample = None self.experimentSetup = None self.integrator_config = {} self.scale = None self.detector = None self.xsDataResult = XSDataResultBioSaxsProcessOneFilev1_0() def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.checkParameters") self.checkMandatoryParameters(self.dataInput, "Data Input is None") self.checkMandatoryParameters(self.dataInput.rawImage, "No raw image provided") self.checkMandatoryParameters(self.dataInput.sample, "No sample information provided") self.checkMandatoryParameters(self.dataInput.experimentSetup, "No experimental setup provided") self.checkMandatoryParameters(self.dataInput.integratedCurve, "Missing integratedCurve") def configure(self): """ Configures the plugin from the configuration file with the following parameters: - DummyPixelValue: the value be assigned to dummy pixels. - DummyPixelDelta: the value be assigned to delta dummy. - DeviceType: "lut_ocl_1,3" will select device #3 on first platform #1 """ EDPluginControl.configure(self) if not self.__configured: with self.semaphore: if not self.__configured: self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.configure") dummy = self.config.get(self.CONF_DUMMY_PIXEL_VALUE) if dummy is None: strMessage = 'EDPluginBioSaxsProcessOneFilev1_4.configure: %s Configuration parameter missing: \ %s, defaulting to "%s"' % (self.getBaseName(), self.CONF_DUMMY_PIXEL_VALUE, self.dummy) self.WARNING(strMessage) self.addErrorWarningMessagesToExecutiveSummary(strMessage) else: self.__class__.dummy = float(dummy) ddummy = self.config.get(self.CONF_DUMMY_PIXEL_DELTA) if ddummy is None: strMessage = 'EDPluginBioSaxsProcessOneFilev1_4.configure: %s Configuration parameter missing: \ %s, defaulting to "%s"' % (self.getBaseName(), self.CONF_DUMMY_PIXEL_DELTA, self.delta_dummy) self.WARNING(strMessage) self.addErrorWarningMessagesToExecutiveSummary(strMessage) else: self.__class__.delta_dummy = float(ddummy) method = self.config.get(self.CONF_OPENCL_DEVICE) if method is None: strMessage = 'EDPluginBioSaxsProcessOneFilev1_4.configure: %s Configuration parameter missing: \ %s, defaulting to "%s"' % (self.getBaseName(), self.CONF_OPENCL_DEVICE, self.METHOD) self.WARNING(strMessage) self.addErrorWarningMessagesToExecutiveSummary(strMessage) else: self.__class__.METHOD = method self.__class__.__configured = True def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.preProcess") self.__edPluginWaitFile = self.loadPlugin(self.cpWaitFile) if self.dataInput.rawImageSize is not None: self.rawImageSize = self.dataInput.rawImageSize if self.dataInput.rawImage is not None: self.rawImage = self.dataInput.rawImage.path.value self.integratedCurve = self.dataInput.integratedCurve.path.value curveDir = os.path.dirname(self.integratedCurve) if not os.path.isdir(curveDir): try: os.mkdir(curveDir) except OSError: # could occur in race condition ... pass self.sample = self.dataInput.sample self.experimentSetup = self.dataInput.experimentSetup self.detector = self.experimentSetup.detector.value if self.detector.lower() == "pilatus": self.detector = "Pilatus1M" else: self.detector = self.detector.capitalize() self.integrator_config = {'dist': self.experimentSetup.detectorDistance.value, 'pixel1': self.experimentSetup.pixelSize_2.value, # flip X,Y 'pixel2': self.experimentSetup.pixelSize_1.value, # flip X,Y 'poni1': self.experimentSetup.beamCenter_2.value * self.experimentSetup.pixelSize_2.value, 'poni2': self.experimentSetup.beamCenter_1.value * self.experimentSetup.pixelSize_1.value, 'rot1': 0.0, 'rot2': 0.0, 'rot3': 0.0, 'splineFile': None, 'detector': self.detector } i0 = self.experimentSetup.beamStopDiode.value if i0 == 0: warn = "beamStopDiode is Null --> If we are testing, this is OK, else investigate !!!" self.lstExecutiveSummary.append(warn) self.warning(warn) self.scale = self.experimentSetup.normalizationFactor.value else: self.scale = self.experimentSetup.normalizationFactor.value / i0 def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.process") xsd = XSDataInputWaitFile(expectedFile=XSDataFile(XSDataString(self.rawImage)), expectedSize=self.rawImageSize, timeOut=XSDataTime(30)) self.__edPluginWaitFile.setDataInput(xsd) self.__edPluginWaitFile.connectSUCCESS(self.doSuccessWaitFile) self.__edPluginWaitFile.connectFAILURE(self.doFailureWaitFile) self.__edPluginWaitFile.executeSynchronous() if self.isFailure(): return self.xsDataResult.sample = self.sample self.xsDataResult.experimentSetup = self.experimentSetup q, I, std = self.integrate() I = self.normalize(I) std = self.normalize(std) self.write3ColumnAscii(q, I, std, self.integratedCurve) self.xsDataResult.dataQ = EDUtilsArray.arrayToXSData(q) self.xsDataResult.dataI = EDUtilsArray.arrayToXSData(I) self.xsDataResult.dataStdErr = EDUtilsArray.arrayToXSData(std) def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.postProcess") # Create some output data if os.path.exists(self.integratedCurve): self.xsDataResult.integratedCurve = XSDataFile(XSDataString(self.integratedCurve)) self.xsDataResult.status = XSDataStatus(executiveSummary=XSDataString(os.linesep.join(self.lstExecutiveSummary))) self.setDataOutput(self.xsDataResult) def integrate(self): img = fabio.open(self.rawImage) if "Date" in img.header: self.experimentSetup.timeOfFrame = XSDataTime(time.mktime(time.strptime(img.header["Date"], "%a %b %d %H:%M:%S %Y"))) wavelength = EDUtilsUnit.getSIValue(self.experimentSetup.wavelength) current_config = self.integrator.getPyFAI() short_config = {} for key in self.integrator_config: short_config[key] = current_config[key] with self.__class__.semaphore: if (short_config != self.integrator_config) or \ (self.integrator.wavelength != wavelength) or\ (self.maskfile != self.experimentSetup.maskFile.path.value): self.screen("Resetting PyFAI integrator") self.integrator.setPyFAI(**self.integrator_config) self.integrator.wavelength = wavelength self.integrator.detector.mask = self.calc_mask() q, I, std = self.integrator.integrate1d(img.data, max(img.dim1, img.dim2), correctSolidAngle=True, dummy=self.dummy, delta_dummy=self.delta_dummy, filename=None, error_model="poisson", radial_range=None, azimuth_range=None, polarization_factor=0, dark=None, flat=None, method=self.METHOD, unit="q_nm^-1", safe=False) self.lstExecutiveSummary.append("Azimuthal integration of raw image '%s'-->'%s'." % (self.rawImage, self.integratedCurve)) return q, I, std def normalize(self, data): """ Perform the normalization of some data @return: normalized data """ maskedData = numpy.ma.masked_array(data, abs(data - self.dummy) < self.delta_dummy) return numpy.ma.filled(maskedData * self.scale, self.dummy) def calc_mask(self): """ Merge the natural mask from the detector with the user proided one. @return: numpy array with the mask """ mask = fabio.open(self.experimentSetup.maskFile.path.value).data detector_mask = pyFAI.detectors.detector_factory(self.detector).calc_mask() shape0, shape1 = detector_mask.shape if detector_mask.shape == mask.shape: mask = numpy.logical_or(mask, detector_mask) else: # crop the user defined mask mask = numpy.logical_or(mask[:shape0, :shape1], detector_mask) self.__class__.maskfile = self.experimentSetup.maskFile.path.value return mask def write3ColumnAscii(self, npaQ, npaI, npaStd=None, outputCurve="output.dat", hdr="#", linesep=os.linesep): """ @param npaQ,npaI,npaStd: 3x 1d numpy array containing Scattering vector, Intensity and deviation @param outputCurve: name of the 3-column ascii file to be written @param hdr: header mark, usually '#' Adam Round explicitelly asked for (email from Date: Tue, 04 Oct 2011 15:22:29 +0200) : Modification from: # BSA buffer # Sample c= 0.0 mg/ml (these two lines are required for current DOS pipeline and can be cleaned up once we use EDNA to get to ab-initio models) # # Sample environment: # Detector = Pilatus 1M # PixelSize_1 = 0.000172 # PixelSize_2 = 6.283185 (I think it could avoid confusion if we give teh actual pixel size as 0.000172 for X and Y and not to give the integrated sizes. Also could there also be a modification for PixelSize_1 as on the diagonal wont it be the hypotenuse (0.000243)? and thus will be on average a bit bigger than 0.000172) # # title = BSA buffer # Frame 7 of 10 # Time per frame (s) = 10 # SampleDistance = 2.43 # WaveLength = 9.31e-11 # Normalization = 0.0004885 # History-1 = saxs_angle +pass -omod n -rsys normal -da 360_deg -odim = 1 /data/id14eh3/inhouse/saxs_pilatus/Adam/EDNAtests/2d/dumdum_008_07.edf/data/id14eh3/inhouse/saxs_pilatus/Adam/EDNAtests/misc/dumdum_008_07.ang # DiodeCurr = 0.0001592934 # MachCurr = 163.3938 # Mask = /data/id14eh3/archive/CALIBRATION/MASK/Pcon_01Jun_msk.edf # SaxsDataVersion = 2.40 # # N 3 # L q*nm I_BSA buffer stddev # # Sample Information: # Storage Temperature (degrees C): 4 # Measurement Temperature (degrees C): 10 # Concentration: 0.0 # Code: BSA s-vector Intensity Error s-vector Intensity Error s-vector Intensity Error s-vector Intensity Error """ hdr = str(hdr) headers = [] if self.sample.comments is not None: headers.append(hdr + " " + self.sample.comments.value) else: headers.append(hdr) if self.sample.concentration is not None: headers.append(hdr + " Sample c= %s mg/ml" % self.sample.concentration.value) else: headers.append(hdr + " Sample c= -1 mg/ml") headers += [hdr, hdr + " Sample environment:"] if self.experimentSetup.detector is not None: headers.append(hdr + " Detector = %s" % self.experimentSetup.detector.value) if self.experimentSetup.pixelSize_1 is not None: headers.append(hdr + " PixelSize_1 = %s" % self.experimentSetup.pixelSize_1.value) if self.experimentSetup.pixelSize_2 is not None: headers.append(hdr + " PixelSize_2 = %s" % self.experimentSetup.pixelSize_2.value) headers.append(hdr) if self.sample.comments is not None: headers.append(hdr + " title = %s" % self.sample.comments.value) if (self.experimentSetup.frameNumber is not None) and\ (self.experimentSetup.frameMax is not None): headers.append(hdr + " Frame %s of %s" % (self.experimentSetup.frameNumber.value, self.experimentSetup.frameMax.value)) if self.experimentSetup.exposureTime is not None: headers.append(hdr + " Time per frame (s) = %s" % self.experimentSetup.exposureTime.value) if self.experimentSetup.detectorDistance is not None: headers.append(hdr + " SampleDistance = %s" % self.experimentSetup.detectorDistance.value) if self.experimentSetup.wavelength is not None: headers.append(hdr + " WaveLength = %s" % self.experimentSetup.wavelength.value) if self.experimentSetup.normalizationFactor is not None: headers.append(hdr + " Normalization = %s" % self.experimentSetup.normalizationFactor.value) if self.experimentSetup.beamStopDiode is not None: headers.append(hdr + " DiodeCurr = %s" % self.experimentSetup.beamStopDiode.value) if self.experimentSetup.machineCurrent is not None: headers.append(hdr + " MachCurr = %s" % self.experimentSetup.machineCurrent.value) if self.experimentSetup.maskFile is not None: headers.append(hdr + " Mask = %s" % self.experimentSetup.maskFile.path.value) headers.append(hdr) headers.append(hdr + " N 3") if self.sample.comments is not None: headers.append(hdr + " L q*nm I_%s stddev" % self.sample.comments.value) else: headers.append(hdr + " L q*nm I_ stddev") headers.append(hdr) headers.append(hdr + " Sample Information:") if self.experimentSetup.storageTemperature is not None: headers.append(hdr + " Storage Temperature (degrees C): %s" % self.experimentSetup.storageTemperature.value) if self.experimentSetup.exposureTemperature is not None: headers.append(hdr + " Measurement Temperature (degrees C): %s" % self.experimentSetup.exposureTemperature.value) if self.sample.concentration is not None: headers.append(hdr + " Concentration: %s" % self.sample.concentration.value) else: headers.append(hdr + " Concentration: -1") if self.sample.code is not None: headers.append(hdr + " Code: %s" % self.sample.code.value) else: headers.append(hdr + " Code: ") with open(outputCurve, "w") as f: f.writelines(linesep.join(headers)) f.write(linesep) if npaStd is None: data = ["%14.6e %14.6e " % (q, I) for q, I in zip(npaQ, npaI) if abs(I - self.dummy) > self.delta_dummy] else: data = ["%14.6e %14.6e %14.6e" % (q, I, std) for q, I, std in zip(npaQ, npaI, npaStd) if abs(I - self.dummy) > self.delta_dummy] data.append("") f.writelines(linesep.join(data)) f.flush() def doSuccessWaitFile(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.doSuccessWaitFile") self.retrieveSuccessMessages(_edPlugin, "EDPluginBioSaxsProcessOneFilev1_4.doSuccessWaitFile") def doFailureWaitFile(self, _edPlugin=None): self.DEBUG("EDPluginBioSaxsProcessOneFilev1_4.doFailureWaitFile") self.retrieveFailureMessages(_edPlugin, "EDPluginBioSaxsProcessOneFilev1_4.doFailureWaitFile") self.lstExecutiveSummary.append("Timeout in waiting for file '%s'" % (self.rawImage)) self.setFailure()
class EDPluginExecMeasureOffsetv1_0(EDPluginExec): """ An exec plugin that taked two images and measures the offset between the two. """ __sem = Semaphore() CONF_CONVOLUTION = None CONF_CONVOLUTION_KEY = "convolution" CONF_CONVOLUTION_DEFAULT = "numpy" #can be "fftw", "cuda" or simply falls back on numpy.fftpack def __init__(self): """ """ EDPluginExec.__init__(self) self.setXSDataInputClass(XSDataInputMeasureOffset) self.npaIm1 = None self.npaIm2 = None self.tOffset = None self.tCrop = None self.tCenter = None self.tWidth = None self.tSmooth = None self.bBackgroundsubtraction = False self.sobel = False def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginExecMeasureOffsetv1_0.checkParameters") self.checkMandatoryParameters(self.dataInput, "Data Input is None") def configure(self): """ Configures the plugin from the configuration file with the following parameters - The CONVOLUTION method """ EDPluginExec.configure(self) if self.CONF_CONVOLUTION is None: with self.__class__.__sem: self.DEBUG("EDPluginExecMeasureOffsetv1_0.configure") xsPluginItem = self.getConfiguration() if xsPluginItem is None: self.WARNING( "EDPluginExecMeasureOffsetv1_0.configure: No plugin item defined." ) xsPluginItem = XSPluginItem() strCONVOLUTION = EDConfiguration.getStringParamValue( xsPluginItem, self.CONF_CONVOLUTION_KEY) if strCONVOLUTION is None: self.WARNING("EDPluginExecMeasureOffsetv1_0.configure: No configuration parameter found for: %s using default value: %s\n%s"\ % (self.CONF_CONVOLUTION_KEY, self.CONF_CONVOLUTION_DEFAULT, xsPluginItem.marshal())) self.__class__CONF_CONVOLUTION = self.CONF_CONVOLUTION_DEFAULT else: self.__class__.CONF_CONVOLUTION = strCONVOLUTION.strip( ).lower() def preProcess(self, _edObject=None): EDPluginExec.preProcess(self) self.DEBUG("EDPluginExecMeasureOffsetv1_0.preProcess") sdi = self.dataInput images = sdi.image arrays = sdi.array if len(images) == 2: self.npaIm1 = numpy.array(EDUtilsArray.getArray(images[0])) self.npaIm2 = numpy.array(EDUtilsArray.getArray(images[1])) elif len(arrays) == 2: self.npaIm1 = EDUtilsArray.xsDataToArray(arrays[0]) self.npaIm2 = EDUtilsArray.xsDataToArray(arrays[1]) else: strError = "EDPluginExecMeasureOffsetv1_0.preProcess: You should either provide two images or two arrays, but I got: %s" % sdi.marshal( )[:1000] self.ERROR(strError) self.setFailure() raise RuntimeError(strError) crop = sdi.cropBorders if len(crop) > 1: self.tCrop = tuple([i.value for i in crop]) elif len(crop) == 1: self.tCrop = (crop[0].value, crop[0].value) center = sdi.center if len(center) > 1: self.tCenter = tuple([i.value for i in center]) elif len(center) == 1: self.tCenter = (center[0].value, center[0].value) width = sdi.width if len(width) > 1: self.tWidth = tuple([i.value for i in width]) elif len(width) == 1: self.tWidth = (width[0].value, width[0].value) smooth = sdi.smoothBorders if len(smooth) == 2: self.tSmooth = (smooth[0].value, smooth[1].value) elif len(smooth) == 1: self.tSmooth = (smooth[0].value, smooth[0].value) if sdi.backgroundSubtraction is not None: self.bBackgroundsubtraction = (sdi.backgroundSubtraction.value in [1, True, "true"]) if sdi.sobelFilter is not None: self.sobel = (sdi.sobelFilter in [1, True, "true"]) EDAssert.equal(self.npaIm1.shape, self.npaIm2.shape, "Images have the same size") def process(self, _edObject=None): EDPluginExec.process(self) self.DEBUG("EDPluginExecMeasureOffsetv1_0.process") shape = self.npaIm1.shape if (self.tCrop is not None): d0min = min(shape[0], self.tCrop[0]) d1min = min(shape[1], self.tCrop[1]) d0max = max(0, shape[0] - self.tCrop[0]) d1max = max(0, shape[1] - self.tCrop[1]) shape = (d0max - d0min, d1max - d1min) else: d0min = 0 d0max = shape[0] d1min = 0 d1max = shape[1] if self.tCenter is None: #the default center is the geometry center of the image ... self.tCenter = [i // 2 for i in shape] if self.tWidth is not None: d0min = max(0, self.tCenter[0] - (self.tWidth[0] // 2)) d0max = min(shape[0], d0min + self.tWidth[0]) d1min = max(0, self.tCenter[1] - (self.tWidth[1] // 2)) d1max = min(shape[1], d1min + self.tWidth[1]) shape = (d0max - d0min, d1max - d1min) if shape != self.npaIm1.shape: self.DEBUG( "Redefining ROI to %s - %s ; %s - %s as crop=%s, center=%s and width=%s" % (d0min, d0max, d1min, d1max, self.tCrop, self.tCenter, self.tWidth)) self.npaIm1 = self.npaIm1[d0min:d0max, d1min:d1max] self.npaIm2 = self.npaIm2[d0min:d0max, d1min:d1max] shape = self.npaIm1.shape self.DEBUG("After Crop, images have shape : %s and %s " % (self.npaIm1.shape, self.npaIm2.shape)) if self.bBackgroundsubtraction: self.DEBUG("performing background subtraction") x = range(shape[0] - 1) + [shape[0] - 1] * (shape[0] - 1) + range( shape[0] - 1, 0, -1) + [0] * (shape[0] - 1) y = [0] * (shape[1] - 1) + range(shape[1] - 1) + [shape[1] - 1] * ( shape[1] - 1) + range(shape[1] - 1, 0, -1) spline1 = scipy.interpolate.SmoothBivariateSpline( x, y, [self.npaIm1[x[i], y[i]] for i in xrange(len(x))], kx=1, ky=1) spline2 = scipy.interpolate.SmoothBivariateSpline( x, y, [self.npaIm2[x[i], y[i]] for i in xrange(len(x))], kx=1, ky=1) self.npaIm1 -= spline1(range(shape[0]), range(shape[1])) self.npaIm2 -= spline2(range(shape[0]), range(shape[1])) if self.tSmooth is not None: self.npaIm1 *= MeasureOffset.make_mask(shape, self.tSmooth) self.npaIm2 *= MeasureOffset.make_mask(shape, self.tSmooth) if self.sobel: self.npaIm1 = scipy.ndimage.sobel(self.npaIm1) self.npaIm2 = scipy.ndimage.sobel(self.npaIm2) self.DEBUG("Doing ffts using %s" % self.CONF_CONVOLUTION) offset, logs = MeasureOffset.measure_offset(self.npaIm1, self.npaIm2, self.CONF_CONVOLUTION, withLog=True) self.tOffset = [XSDataDouble(f) for f in offset] logs.insert(0, "") if len(logs) <= 4: self.DEBUG(os.linesep.join(logs)) else: self.WARNING(os.linesep.join(logs)) def postProcess(self, _edObject=None): EDPluginExec.postProcess(self) self.DEBUG("EDPluginExecMeasureOffsetv1_0.postProcess") # Create some output data self.setDataOutput(XSDataResultMeasureOffset(offset=self.tOffset))
class EDPluginExecNormalizeImagev1_1(EDPluginExec): """ Plugin that normalizes an image (subtract dark current and divide by flat-field image, taking into account the exposure time) v1.1 is compatible with EDShare -> lower memory footprint """ dictDark = {} #Nota: the key is a string: str(exposutTime) # listDarkExposure = [] # listDarkArray = [] semaphore = Semaphore() dtype = None # CONF_DTYPE_KEY = "dtype" CONF_DTYPE_DEFAULT = "float32" def __init__(self): """ Constructor of the plugin """ EDPluginExec.__init__(self) self.setXSDataInputClass(XSDataInputNormalize) self.listDataArray = [] self.listDataExposure = [] self.listDarkArray = [] self.listDarkExposure = [] self.listFlatArray = [] self.listFlatExposure = [] self.npaNormalized = None self.strOutputFilename = None self.strOutputShared = None self.shape = None def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginExecNormalizeImagev1_1.checkParameters") self.checkMandatoryParameters(self.getDataInput(), "Data Input is None") def configure(self): """ Add the dtype to the configuration file, probably "float32" or "float64" """ EDPluginExec.configure(self) self.DEBUG("EDPluginExecNormalizeImagev1_1.configure") if self.__class__.dtype is None: with self.__class__.semaphore: xsPluginItem = self.getConfiguration() if (xsPluginItem == None): self.WARNING("EDPluginExecNormalizeImagev1_1.configure: No plugin item defined.") xsPluginItem = XSPluginItem() strDtype = EDConfiguration.getStringParamValue(xsPluginItem, self.CONF_DTYPE_KEY) if(strDtype == None): self.WARNING("EDPluginExecNormalizeImagev1_1.configure: No configuration parameter found for: %s using default value: %s\n%s"\ % (self.CONF_DTYPE_KEY, self.CONF_DTYPE_DEFAULT, xsPluginItem.marshal())) self.__class__.dtype = self.CONF_DTYPE_DEFAULT else: self.__class__.dtype = str(strDtype.strip().lower()) def preProcess(self, _edObject=None): EDPluginExec.preProcess(self) self.DEBUG("EDPluginExecNormalizeImagev1_1.preProcess") sdi = self.getDataInput() if sdi.getData() == []: strError = "You should either provide at least ONE input filename or an array, you provided: %s" % sdi.marshal() self.ERROR(strError) self.setFailure() raise RuntimeError(strError) else: for inputData in sdi.getData(): if inputData.exposureTime is None: self.WARNING("You did not provide an exposure time for DATA... using default: 1") self.listDataExposure.append(1.0) else: self.listDataExposure.append(EDUtilsUnit.getSIValue(inputData.exposureTime)) if inputData.path is not None: strPath = inputData.path.value if os.path.isfile(strPath): self.listDataArray.append(fabio.open(strPath).data) else: strError = "The input file provided for DATA is not a valid file: %s" % strPath self.ERROR(strError) self.setFailure() raise RuntimeError(strError) elif inputData.array is not None: self.listDataArray.append(EDUtilsArray.xsDataToArray(inputData.array)) else: strError = "You should either provide an input filename or an array for DATA, you provided: %s" % inputData.marshal() self.ERROR(strError) self.setFailure() raise RuntimeError(strError) for inputFlat in sdi.getFlat(): if inputFlat.exposureTime is None: self.WARNING("You did not provide an exposure time for FLAT... using default: 1") expTime = 1.0 else: expTime = EDUtilsUnit.getSIValue(inputFlat.exposureTime) self.listFlatExposure.append(expTime) if inputFlat.path is not None: strPath = inputFlat.path.value if os.path.isfile(strPath): self.listFlatArray.append(fabio.open(strPath).data) else: strError = "The input file provided for FLAT is not a valid file: %s" % strPath self.ERROR(strError) self.setFailure() raise RuntimeError(strError) elif inputFlat.array is not None: self.listFlatArray.append(EDUtilsArray.xsDataToArray(inputFlat.array)) else: strError = "You should either provide an input filename or an array for FLAT, you provided: %s" % inputFlat.marshal() self.ERROR(strError) self.setFailure() raise RuntimeError(strError) with EDPluginExecNormalizeImagev1_1.semaphore: for inputDark in sdi.getDark(): if inputDark.exposureTime is None: self.WARNING("You did not provide an exposure time for Dark... using default: 1") expTime = 1.0 else: expTime = EDUtilsUnit.getSIValue(inputDark.exposureTime) strMeanDarkKey = "/".join((self.getClassName(), "MeanDark%6.3f" % expTime)) if strMeanDarkKey not in EDShare: self.listDarkExposure.append(expTime) if inputDark.path is not None: strPath = inputDark.path.value if os.path.isfile(strPath): self.listDarkArray.append(fabio.open(strPath).data) else: strError = "The input file provided for Dark is not a valid file: %s" % strPath self.ERROR(strError) self.setFailure() raise RuntimeError(strError) elif inputDark.array is not None: self.listDarkArray.append(EDUtilsArray.xsDataToArray(inputDark.array)) elif inputDark.shared is not None: self.listDarkArray.append(EDShare[inputDark.shared.value]) else: strError = "You should either provide an input filename or an array for Dark, you provided: %s" % inputDark.marshal() self.ERROR(strError) self.setFailure() raise RuntimeError(strError) if sdi.output is not None: if (sdi.output.path is not None): self.strOutputFilename = sdi.output.path.value elif (sdi.output.shared is not None): self.strOutputShared = sdi.output.shared.value # else export as array. EDAssert.equal(len(self.listDataArray), len(self.listDataExposure), _strComment="number of data images / exposure times ") EDAssert.equal(len(self.listFlatArray), len(self.listFlatExposure), _strComment="number of flat images / exposure times ") EDAssert.equal(len(self.listDarkArray), len(self.listDarkExposure), _strComment="number of dark images / exposure times ") def process(self, _edObject=None): EDPluginExec.process(self) self.DEBUG("EDPluginExecNormalizeImagev1_1.process") #numerator part: fTotalDataTime = 0.0 self.shape = self.listDataArray[0].shape npaSummedData = numpy.zeros(self.shape, dtype=self.dtype) for i in range(len(self.listDataArray)): fTotalDataTime += self.listDataExposure[i] npaSummedData += numpy.maximum(self.listDataArray[i] - self.getMeanDark(self.listDataExposure[i]), 0) npaNormalizedData = (npaSummedData / fTotalDataTime).astype(self.dtype) #denominator part fTotalFlatTime = 0.0 npaSummedFlat = numpy.zeros(self.shape, dtype=self.dtype) for i in range(len(self.listFlatArray)): fTotalFlatTime += self.listFlatExposure[i] npaSummedFlat += numpy.maximum(self.listFlatArray[i] - self.getMeanDark(self.listFlatExposure[i]), 0) npaNormalizedFlat = (npaSummedFlat / fTotalFlatTime).astype(self.dtype) self.npaNormalized = npaNormalizedData / numpy.maximum(npaNormalizedFlat, numpy.ones_like(npaNormalizedFlat)) if self.npaNormalized.dtype != numpy.dtype(self.dtype): self.npaNormalized = self.npaNormalized.astype(self.dtype) def postProcess(self, _edObject=None): EDPluginExec.postProcess(self) self.DEBUG("EDPluginExecNormalizeImagev1_1.postProcess") xsDataResult = XSDataResultNormalize() if self.strOutputFilename is not None: self.DEBUG("Writing file %s" % self.strOutputFilename) edf = fabio.edfimage.edfimage(data=self.npaNormalized, header={}) edf.write(self.strOutputFilename) xsdo = XSDataImageExt(path=XSDataString(self.strOutputFilename)) elif self.strOutputShared is not None: self.DEBUG("EDShare --> %s" % self.strOutputShared) EDShare[ self.strOutputShared] = self.npaNormalized xsdo = XSDataImageExt(shared=XSDataString(self.strOutputShared)) else: xsdo = XSDataImageExt(array=EDUtilsArray.arrayToXSData(self.npaNormalized)) xsDataResult.output = xsdo # Create some output data self.setDataOutput(xsDataResult) def finallyProcess(self, _edObject=None): """ after processing of the plugin: remove reference to large objects to save memory """ self.DEBUG("EDPluginExecNormalizeImagev1_1.finallyProcess") EDPluginExec.finallyProcess(self, _edObject) self.listDataArray = None self.listDataExposure = None self.listFlatArray = None self.listFlatExposure = None self.npaNormalized = None def getMeanDark(self, _fExposureTime): """ @param _fExposureTime: exposure time @return: mean of darks with this exposure time """ with self.__class__.semaphore: if str(_fExposureTime) not in self.__class__.dictDark: npaSumDark = numpy.zeros(self.shape, dtype="float64") count = 0 for fExpTime, npaDark in zip(self.listDarkExposure, self.listDarkArray): if abs(fExpTime - _fExposureTime) / _fExposureTime < 1e-4: npaSumDark += npaDark count += 1 if count == 0: self.WARNING("No Dark image with Exposure time = %.3f, no dark subtraction" % _fExposureTime) self.__class__.dictDark[str(_fExposureTime) ] = npaSumDark else: self.__class__.dictDark[str(_fExposureTime) ] = npaSumDark / float(count) return self.__class__.dictDark[str(_fExposureTime)]
class CudaCorrelate(object): plans = {} data1_gpus = {} data2_gpus = {} multconj = None ctx = None # pycuda.autoinit.context.pop() # ctx.pop() sem = Semaphore() initsem = Semaphore() def __init__(self, shape): self.shape = tuple(shape) def init(self): if self.ctx is None: with self.__class__.initsem: if self.ctx is None: self.__class__.ctx = pycuda.autoinit.context if not self.shape in self.plans: with self.__class__.initsem: if not self.shape in self.plans: self.ctx.push() if not self.__class__.multconj: self.__class__.multconj = pycuda.elementwise.ElementwiseKernel( "pycuda::complex<double> *a, pycuda::complex<double> *b", "a[i]*=conj(b[i])") if self.shape not in self.__class__.data1_gpus: self.__class__.data1_gpus[self.shape] = gpuarray.empty( self.shape, numpy.complex128) if self.shape not in self.__class__.data2_gpus: self.__class__.data2_gpus[self.shape] = gpuarray.empty( self.shape, numpy.complex128) if self.shape not in self.__class__.plans: self.__class__.plans[self.shape] = cu_fft.Plan( self.shape, numpy.complex128, numpy.complex128) self.ctx.synchronize() self.ctx.pop() @classmethod def clean(cls): with initsem: with sem: if self.ctx: cls.ctx.push() for plan_name in list(cls.plans.keys()): plan = cls.plans.pop(plan_name) del plan for plan_name in list(cls.data1_gpus.keys()): data = cls.data1_gpus.pop(plan_name) data.gpudata.free() del data for plan_name in cls.data2_gpus.copy(): data = cls.data2_gpus.pop(plan_name) data.gpudata.free() del data cls.ctx.pop() cls.ctx = None def correlate(self, data1, data2): self.init() with self.__class__.sem: self.ctx.push() plan = self.__class__.plans[self.shape] data1_gpu = self.__class__.data1_gpus[self.shape] data2_gpu = self.__class__.data2_gpus[self.shape] data1_gpu.set(data1.astype(numpy.complex128)) cu_fft.fft(data1_gpu, data1_gpu, plan) data2_gpu.set(data2.astype(numpy.complex128)) cu_fft.fft(data2_gpu, data2_gpu, plan) # data1_gpu *= data2_gpu.conj() self.multconj(data1_gpu, data2_gpu) cu_fft.ifft(data1_gpu, data1_gpu, plan, True) # self.ctx.synchronize() res = data1_gpu.get().real self.ctx.pop() return res
class EDPluginControlAlignStackv1_0(EDPluginControl): """ This control plugin aligns a stack on images in a single HDF5 3D-structure """ __iRefFrame = None __dictRelShift = { } #key=frame number N, value= 2-tuple of shift relative to frame N-1 __dictAbsShift = { } #key=frame number N, value= 2-tuple of shift relative to frame iRefFrame __semaphore = Semaphore() MaxOffset = None def __init__(self): """ """ EDPluginControl.__init__(self) self.setXSDataInputClass(XSDataInputAlignStack) self.iFrames = [] self.npArrays = [] self.xsdMeasureOffset = None self.hdf5ExtraAttributes = None self.xsdHDF5File = None self.xsdHDF5Internal = None self.bAlwaysMOvsRef = True self.bDoAlign = True self.semAccumulator = Semaphore() self.semMeasure = Semaphore() self.semShift = Semaphore() self.lstSem = [ self.locked(), self.semAccumulator, self.semMeasure, self.semShift ] self.queue = Queue() self.__strControlledPluginAccumulator = "EDPluginAccumulatorv1_0" self.__strControlledPluginMeasureFFT = "EDPluginExecMeasureOffsetv1_0" self.__strControlledPluginMeasureSIFT = "EDPluginExecMeasureOffsetv2_0" self.__strControlledPluginShift = "EDPluginExecShiftImagev1_1" self.__strControlledPluginHDF5 = "EDPluginHDF5StackImagesv10" def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginControlAlignStackv1_0.checkParameters") self.checkMandatoryParameters(self.dataInput, "Data Input is None") self.checkMandatoryParameters(self.dataInput.getHDF5File(), "No Output HDF5 file provided") self.checkMandatoryParameters(self.dataInput.getInternalHDF5Path(), "No output HDF5 internal path provided") self.checkMandatoryParameters(self.dataInput.images, "No images to process provided") def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlAlignStackv1_0.preProcess") sdi = self.dataInput self.xsdHDF5File = sdi.HDF5File self.xsdHDF5Internal = sdi.internalHDF5Path self.hdf5ExtraAttributes = sdi.extraAttributes if sdi.dontAlign is not None: self.bDoAlign = not (bool(sdi.dontAlign.value)) self.iFrames = [xsd.value for xsd in sdi.index] for idx, oneXSDFile in enumerate(sdi.images): self.npArrays.append(EDUtilsArray.getArray(oneXSDFile)) if len(self.iFrames) <= idx: if (oneXSDFile.number is not None): self.iFrames.append(oneXSDFile.number.value) elif oneXSDFile.path is not None: number = "" filename = oneXSDFile.path.value basename = os.path.splitext(filename)[0] for i in basename[-1:0:-1]: if i.isdigit(): number = i + number else: break self.iFrames.append(int(number)) if self.npArrays == []: strError = "EDPluginControlAlignStackv1_0.preProcess: You should either provide an images or an arrays, but I got: %s" % sdi.marshal( ) self.ERROR(strError) self.setFailure() self.xsdMeasureOffset = sdi.measureOffset if self.xsdMeasureOffset.alwaysVersusRef is not None: self.bAlwaysMOvsRef = bool( self.xsdMeasureOffset.alwaysVersusRef.value) with self.__class__.__semaphore: if (self.__class__.__iRefFrame is None): self.DEBUG("reference Frame is: %s" % sdi.frameReference.value) if sdi.getFrameReference() is not None: self.__class__.__iRefFrame = sdi.frameReference.value else: self.__class__.__iRefFrame = 0 if len(self.iFrames) == len(self.npArrays): for i, j in zip(self.iFrames, self.npArrays): self.addFrame(i, j) else: self.ERROR( "EDPluginControlAlignStackv1_0.preProcess: You should either provide an images with a frame number or precise it in the XML ! I got: %s" % sdi.marshal()) self.setFailure() raise RuntimeError def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlAlignStackv1_0.process") for iFrame in self.iFrames: edPluginExecAccumulator = self.loadPlugin( self.__strControlledPluginAccumulator) queryRaw = XSDataQuery() queryShift = XSDataQuery() queryRaw.setRemoveItems(XSDataBoolean(False)) queryShift.setRemoveItems(XSDataBoolean(False)) xsdataAcc = XSDataInputAccumulator() if (EDPluginControlAlignStackv1_0.__iRefFrame == iFrame) or (self.bDoAlign == False): EDPluginControlAlignStackv1_0.__dictAbsShift[iFrame] = (0.0, 0.0) EDPluginControlAlignStackv1_0.__dictRelShift[iFrame] = (0.0, 0.0) self.hdf5_offset(index=iFrame, offset=[0.0, 0.0]) edPluginExecShift = self.loadPlugin( self.__strControlledPluginShift) xsdata = XSDataInputShiftImage( index=XSDataInteger(iFrame), offset=[ XSDataDouble(i) for i in EDPluginControlAlignStackv1_0.__dictAbsShift[iFrame] ], inputImage=self.getFrameRef(iFrame), outputImage=XSDataImageExt( shared=XSDataString("Shifted-%06i" % iFrame))) edPluginExecShift.setDataInput(xsdata) edPluginExecShift.connectSUCCESS(self.doSuccessExecShiftImage) edPluginExecShift.connectFAILURE(self.doFailureExecShiftImage) self.queue.put(edPluginExecShift) if (self.bDoAlign == False): self.executeControlledPlugins() return elif EDPluginControlAlignStackv1_0.__iRefFrame < iFrame: if self.bAlwaysMOvsRef: queryRaw.setItem([ XSDataString( "raw %04i" % (EDPluginControlAlignStackv1_0.__iRefFrame)), XSDataString("raw %04i" % iFrame) ]) xsdataAcc.setQuery([queryRaw]) else: queryRaw.setItem([ XSDataString("raw %04i" % (iFrame - 1)), XSDataString("raw %04i" % iFrame) ]) queryShift.setItem([ XSDataString("shift %04i" % i) for i in range( EDPluginControlAlignStackv1_0.__iRefFrame + 1, iFrame + 1) ]) xsdataAcc.setQuery([queryRaw, queryShift]) elif EDPluginControlAlignStackv1_0.__iRefFrame > iFrame: if self.bAlwaysMOvsRef: queryRaw.setItem([ XSDataString("raw %04i" % iFrame), XSDataString( "raw %04i" % (EDPluginControlAlignStackv1_0.__iRefFrame)) ]) xsdataAcc.setQuery([queryRaw]) else: queryRaw.setItem([ XSDataString("raw %04i" % (iFrame + 1)), XSDataString("raw %04i" % iFrame) ]) queryShift.setItem([ XSDataString("shift %04i" % i) for i in range( EDPluginControlAlignStackv1_0.__iRefFrame - 1, iFrame - 1, -1) ]) xsdataAcc.setQuery([queryRaw, queryShift]) if (EDPluginControlAlignStackv1_0.__iRefFrame == iFrame): self.saveReferenceFrame(iFrame) xsdataAcc.setItem([XSDataString("raw %04i" % iFrame)]) edPluginExecAccumulator.setDataInput(xsdataAcc) edPluginExecAccumulator.connectSUCCESS( self.doSuccessExecAccumultor) edPluginExecAccumulator.connectFAILURE( self.doFailureExecAccumulator) self.queue.put(edPluginExecAccumulator) self.executeControlledPlugins() def executeControlledPlugins(self): """ Execute all plugins under control: """ bAllFinished = False while not bAllFinished: #acquire all semaphores to be sure no plugins are under configuration ! for sem in self.lstSem: with sem: pass if self.queue.empty(): self.synchronizePlugins() bAllFinished = self.queue.empty() else: while not self.queue.empty(): try: plugin = self.queue.get() except Exception: self.WARNING( "In EDPluginControlAlignStackv1_0, exception in self.queue.get()" ) break else: #this is a hack to prevent thousands of threads to be launched at once. with EDUtilsParallel.getSemaphoreNbThreads(): pass plugin.execute() self.synchronizePlugins() def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginControlAlignStackv1_0.postProcess") self.executeControlledPlugins() self.synchronizePlugins() # Create some output data xsDataResult = XSDataResultAlignStack() xsDataResult.setHDF5File(self.xsdHDF5File) xsDataResult.setInternalHDF5Path(self.xsdHDF5Internal) self.setDataOutput(xsDataResult) self.emptyListOfLoadedPlugin() def doSuccessExecMeasureOffset(self, _edPlugin=None): with self.semMeasure: self.DEBUG( "EDPluginControlAlignStackv1_0.doSuccessExecMeasureOffset") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlAlignStackv1_0.doSuccessExecMeasureOffset") listIndex = [i.getValue() for i in _edPlugin.dataInput.index] listIndex.sort() dataOutput = _edPlugin.dataOutput if self.bAlwaysMOvsRef: if min(listIndex) < EDPluginControlAlignStackv1_0.__iRefFrame: iToShift, iRef = tuple(listIndex) EDPluginControlAlignStackv1_0.__dictAbsShift[ iToShift] = tuple( [-i.getValue() for i in dataOutput.getOffset()]) else: iRef, iToShift = tuple(listIndex) EDPluginControlAlignStackv1_0.__dictAbsShift[ iToShift] = tuple( [i.getValue() for i in dataOutput.getOffset()]) self.screen( "Frame number %i has absolute offset of %.3f,%.3f" % (iToShift, EDPluginControlAlignStackv1_0.__dictAbsShift[iToShift][0], EDPluginControlAlignStackv1_0.__dictAbsShift[iToShift][1]) ) edPluginExecShift = self.loadPlugin( self.__strControlledPluginShift) xsdata = XSDataInputShiftImage( index=XSDataInteger(iToShift), offset=[ XSDataDouble(i) for i in EDPluginControlAlignStackv1_0.__dictAbsShift[iToShift] ], inputImage=self.getFrameRef(iToShift), outputImage=XSDataImageExt( shared=XSDataString("Shifted-%06i" % iToShift))) edPluginExecShift.setDataInput(xsdata) edPluginExecShift.connectSUCCESS(self.doSuccessExecShiftImage) edPluginExecShift.connectFAILURE(self.doFailureExecShiftImage) self.queue.put(edPluginExecShift) else: if min(listIndex) < EDPluginControlAlignStackv1_0.__iRefFrame: iToShift, iRef = tuple(listIndex) EDPluginControlAlignStackv1_0.__dictRelShift[ iToShift] = tuple( [-i.value for i in dataOutput.offset]) else: iRef, iToShift = tuple(listIndex) EDPluginControlAlignStackv1_0.__dictRelShift[ iToShift] = tuple([i.value for i in dataOutput.offset]) self.screen( "Frame number %i has relative offset of %.3f,%.3f" % (iToShift, EDPluginControlAlignStackv1_0.__dictRelShift[iToShift][0], EDPluginControlAlignStackv1_0.__dictRelShift[iToShift][1]) ) xsdata = XSDataInputAccumulator( item=[XSDataString("shift %04i" % iToShift)]) edPluginExecAccumulator = self.loadPlugin( self.__strControlledPluginAccumulator) edPluginExecAccumulator.setDataInput(xsdata) edPluginExecAccumulator.connectSUCCESS( self.doSuccessExecAccumultor) edPluginExecAccumulator.connectFAILURE( self.doFailureExecAccumulator) self.queue.put(edPluginExecAccumulator) def doFailureExecMeasureOffset(self, _edPlugin=None): self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecMeasureOffset") self.retrieveFailureMessages( _edPlugin, "EDPluginControlAlignStackv1_0.doFailureExecMeasureOffset") self.ERROR( "Failure in execution of the MeasureOffset with input: %s and output %s" % (_edPlugin.dataInput.marshal()[:1000], _edPlugin.dataOutput.marshal()[:1000])) self.setFailure() def doSuccessExecShiftImage(self, _edPlugin=None): with self.semShift: edPluginExecHDF5 = self.loadPlugin(self.__strControlledPluginHDF5) self.DEBUG("EDPluginControlAlignStackv1_0.doSuccessExecShiftImage") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlAlignStackv1_0.doSuccessExecShiftImage") xsdIdx = _edPlugin.dataInput.index self.__class__.MaxOffset = _edPlugin.MAX_OFFSET_VALUE self.hdf5_offset( index=xsdIdx.value, offset=[i.value for i in _edPlugin.dataInput.offset]) xsdata = XSDataInputHDF5StackImages( chunkSegmentation=XSDataInteger(32), forceDtype=XSDataString("float32"), extraAttributes=self.hdf5ExtraAttributes, internalHDF5Path=self.xsdHDF5Internal, HDF5File=self.xsdHDF5File, index=[xsdIdx], inputImageFile=[_edPlugin.dataOutput.outputImage]) # inputArray=[_edPlugin.dataOutput.outputArray]) edPluginExecHDF5.setDataInput(xsdata) edPluginExecHDF5.connectSUCCESS(self.doSuccessExecStackHDF5) edPluginExecHDF5.connectFAILURE(self.doFailureExecStackHDF5) self.queue.put(edPluginExecHDF5) def doFailureExecShiftImage(self, _edPlugin=None): self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecShiftImage") self.retrieveFailureMessages( _edPlugin, "EDPluginControlAlignStackv1_0.doFailureExecShiftImage") self.ERROR( "Failure in execution of the ExecShiftImage with input: %s and output %s" % (_edPlugin.dataInput.marshal()[:1000], _edPlugin.dataOutput.marshal()[:1000])) self.setFailure() def doSuccessExecStackHDF5(self, _edPlugin=None): self.DEBUG("EDPluginControlAlignStackv1_0.doSuccessExecStackHDF5") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlAlignStackv1_0.doSuccessExecStackHDF5") def doFailureExecStackHDF5(self, _edPlugin=None): self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecStackHDF5") self.retrieveFailureMessages( _edPlugin, "EDPluginControlAlignStackv1_0.doFailureExecStackHDF5") self.ERROR( "Failure in execution of the ExecStackHDF5 with input: %s " % (_edPlugin.dataInput.marshal()[:1000])) if _edPlugin.dataOutput is not None: self.ERROR( "Failure in execution of the ExecStackHDF5 with output %s" % (_edPlugin.dataOutput.marshal()[:1000])) def doSuccessExecAccumultor(self, _edPlugin=None): with self.semAccumulator: self.DEBUG("EDPluginControlAlignStackv1_0.doSuccessExecAccumultor") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlAlignStackv1_0.doSuccessExecAccumultor") for query in _edPlugin.dataOutput.getQuery(): self.addExtraTime(60) _edPlugin.addExtraTime(60) accType = query.getItem()[0].getValue().split()[0] listInt = [ int(i.getValue().split()[1]) for i in query.getItem() ] if accType == "raw": listFrame = [self.getFrameRef(i) for i in listInt] xsdata = XSDataInputMeasureOffset(image=listFrame) doSIFT = False if self.xsdMeasureOffset is not None: xsdata.setCropBorders( self.xsdMeasureOffset.getCropBorders()) xsdata.setSmoothBorders( self.xsdMeasureOffset.getSmoothBorders()) xsdata.setBackgroundSubtraction( self.xsdMeasureOffset.getRemoveBackground()) if self.xsdMeasureOffset.useSift is not None: doSIFT = self.xsdMeasureOffset.useSift.value if max(listInt ) > EDPluginControlAlignStackv1_0.__iRefFrame: listInt.sort() else: listInt.sort(reverse=True) xsdata.setIndex([XSDataInteger(i) for i in listInt]) if doSIFT: edPluginExecMeasure = self.loadPlugin( self.__strControlledPluginMeasureSIFT) else: edPluginExecMeasure = self.loadPlugin( self.__strControlledPluginMeasureFFT) edPluginExecMeasure.setDataInput(xsdata) edPluginExecMeasure.connectSUCCESS( self.doSuccessExecMeasureOffset) edPluginExecMeasure.connectFAILURE( self.doFailureExecMeasureOffset) self.queue.put(edPluginExecMeasure) elif accType == "shift": shift_1 = 0.0 shift_2 = 0.0 for frame in listInt: shift_1 += EDPluginControlAlignStackv1_0.__dictRelShift[ frame][0] shift_2 += EDPluginControlAlignStackv1_0.__dictRelShift[ frame][1] if listInt[0] > EDPluginControlAlignStackv1_0.__iRefFrame: iFrameShift = max(listInt) else: iFrameShift = min(listInt) EDPluginControlAlignStackv1_0.__dictAbsShift[ iFrameShift] = (shift_1, shift_2) self.screen( "Frame number %i has absolute offset of %.3f,%.3f" % (iFrameShift, shift_1, shift_2)) edPluginExecShift = self.loadPlugin( self.__strControlledPluginShift) edPluginExecShift.dataInput = XSDataInputShiftImage( index=XSDataInteger(iFrameShift), offset=[XSDataDouble(shift_1), XSDataDouble(shift_2)], inputImage=self.getFrameRef(iFrameShift), outputImage=XSDataImageExt( shared=XSDataString("Shifted-%06i" % iFrameShift))) edPluginExecShift.connectSUCCESS( self.doSuccessExecShiftImage) edPluginExecShift.connectFAILURE( self.doFailureExecShiftImage) self.queue.put(edPluginExecShift) self.DEBUG("Items: %s" % EDPluginAccumulatorv1_0.getItems()) self.DEBUG("Queries: %s" % EDPluginAccumulatorv1_0.getQueries()) def doFailureExecAccumulator(self, _edPlugin=None): self.DEBUG("EDPluginControlAlignStackv1_0.doFailureExecAccumulator") self.retrieveFailureMessages( _edPlugin, "EDPluginControlAlignStackv1_0.doFailureExecAccumulator") self.ERROR( "Failure in execution of the accumulator with input: %s and output %s" % (_edPlugin.dataInput.marshal()[:1000], _edPlugin.dataOutput.marshal()[:1000])) self.setFailure() def saveReferenceFrame(self, iFrame): """ Save the reference frame to the HDF5 file @param iFrame: frame number to save as reference frame """ ref = "reference_frame" hdf5file = self.xsdHDF5File.path.value edpluginHDF5 = self.loadPlugin(self.__strControlledPluginHDF5) with edpluginHDF5.getFileLock(hdf5file): #Seems strange to redefine h5Grp but if there is a flush in between: h5Grp could be closed entry = edpluginHDF5.getHDF5File(hdf5file)[ self.xsdHDF5Internal.value] if ref in entry: del entry[ref] entry[ref] = self.getFrame(iFrame) entry[ref].attrs["index"] = iFrame def hdf5_offset(self, index, offset): with EDPluginHDF5.getFileLock(self.xsdHDF5File.path.value): grp = EDPluginHDF5.getHDF5File( self.xsdHDF5File.path.value)[self.xsdHDF5Internal.value] ds = grp["Offsets"] if self.MaxOffset: if "MaxOffset" not in ds.attrs: ds.attrs["MaxOffset"] = self.MaxOffset ds[index, :] = offset @classmethod def showData(cls): EDVerbose.screen("*" * 20 + "EDPluginControlAlignStackv1_0" + "*" * 20) EDVerbose.screen("Reference Frame: %s" % cls.__iRefFrame) if len(cls.__dictRelShift) < len(cls.__dictAbsShift): mydict = cls.__dictAbsShift.copy() else: mydict = cls.__dictRelShift.copy() lstTxt = [] for i in mydict: txt = "Frame %4i:\t" % i rela = cls.__dictRelShift.get(i) abso = cls.__dictAbsShift.get(i) if rela: txt += "relative: (%.3f, %.3f)\t" % rela else: txt += "relative: %12s\t" % rela if abso: txt += "absolute: (%.3f, %.3f)" % abso else: txt += "absolute: %12s" % abso lstTxt.append(txt) EDVerbose.screen(os.linesep.join(lstTxt)) items = EDPluginAccumulatorv1_0.getItems() items.sort() EDVerbose.screen("Items in the accumulator: %s" % (items)) querylist = [" "] + [ str(i) for i in EDPluginAccumulatorv1_0.getQueries().keys() ] EDVerbose.screen("Queries in the accumulator: " + os.linesep.join(querylist)) @classmethod def addFrame(cls, index, value): """ Just store the value to EDShare """ key = "Normalized-%06i" % int(index) if key not in EDShare: EDShare[key] = value @classmethod def getFrame(cls, index): """ Just Retrieves the value from EDShare """ return EDShare["Normalized-%06i" % int(index)] @classmethod def getFrameRef(cls, index): """ Just retrieves the reference in the EDShare store @return: reference to the frame in EDShare @rtype: XSDataImageExt """ return XSDataImageExt(shared=XSDataString("Normalized-%06i" % int(index))) @classmethod def cleanUp(cls): """ Clean up of the dictionary containing images: Left for compatibility reasons """ EDShare.flush()
class EDUtilsParallel(object): """ A class helping to control the workload on a multi-threaded application. """ _iActualNbCPU = None _semaphore = Semaphore() _semaphoreInit = Semaphore() _semaphoreNbThreads = None _iNbThreads = None @classmethod def _detectNumberOfCPUs(cls): """ detect the actual number of CPUs and stores it in a class variable """ with cls._semaphore: if cls._iActualNbCPU is not None: return iNbCPU = None #The best way: using python 2.6 or later try: import multiprocessing iNbCPU = multiprocessing.cpu_count() except (ImportError, NotImplementedError): iNbCPU = None # POSIX if iNbCPU is None: try: iNbCPU = int(os.sysconf('SC_NPROCESSORS_ONLN')) if iNbCPU <= 0: iNbCPU = None except (AttributeError, ValueError): iNbCPU = None # Windows if iNbCPU is None: try: iNbCPU = int(os.environ['NUMBER_OF_PROCESSORS']) if iNbCPU <= 0: iNbCPU = None except (KeyError, ValueError): iNbCPU = None # jython if iNbCPU is None: try: from java.lang import Runtime runtime = Runtime.getRuntime() iNbCPU = runtime.availableProcessors() if iNbCPU <= 0: iNbCPU = None except ImportError: iNbCPU = None # BSD if iNbCPU is None: try: sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'], stdout=subprocess.PIPE) scStdout = sysctl.communicate()[0] iNbCPU = int(scStdout) if iNbCPU <= 0: iNbCPU = None except (OSError, ValueError): iNbCPU = None # Linux if iNbCPU is None: try: iNbCPU = open('/proc/cpuinfo').read().count('processor\t:') if iNbCPU <= 0: iNbCPU = None except IOError: iNbCPU = None # Solaris if iNbCPU is None: try: pseudoDevices = os.listdir('/devices/pseudo/') expr = re.compile('^cpuid@[0-9]+$') iNbCPU = 0 for pd in pseudoDevices: if expr.match(pd) != None: iNbCPU += 1 if iNbCPU <= 0: iNbCPU = None except OSError: iNbCPU = None # Other UNIXes (heuristic) if iNbCPU is None: try: try: dmesg = open('/var/run/dmesg.boot').read() except IOError: dmesgProcess = subprocess.Popen(['dmesg'], stdout=subprocess.PIPE) dmesg = dmesgProcess.communicate()[0] iNbCPU = 0 while '\ncpu' + str(iNbCPU) + ':' in dmesg: iNbCPU += 1 if iNbCPU <= 0: iNbCPU = None except OSError: iNbCPU = None #if nothing else works ... if iNbCPU is None: iNbCPU = 1 cls._iActualNbCPU = iNbCPU @classmethod def detectNumberOfCPUs(cls, _iMaxCPU=sys.maxint, _bForce=False): """ class method : Detects the number of CPUs on a system. Cribbed from pp. found : http://www.boduch.ca/2009/06/python-cpus.html @param _iMaxCPU: the maximum number of CPUs allowed @type _iMaxCPU: integer @param _bForce: force the number of CPUs for debugging @type _bForce: boolean @return: the number of CPUs @rtype: integer """ if not isinstance(_iMaxCPU, int): _iMaxCPU = sys.maxint else: _iMaxCPU = max(1, _iMaxCPU) if isinstance(_bForce, bool): if _bForce: return _iMaxCPU if cls._iActualNbCPU is None: cls._detectNumberOfCPUs() iNbCPU = cls._iActualNbCPU if iNbCPU < 1: return 1 elif iNbCPU > _iMaxCPU: return _iMaxCPU else: return iNbCPU @classmethod def initializeNbThread(cls, _iNbThread=None): """ Class method: Initialises a semaphore with the right number of threads @param _iNbThread: the maximum number of threads concurrently running and CPU intensive @type _iNbThread: integer """ with cls._semaphoreInit: if cls._semaphoreNbThreads is None: if _iNbThread is None: _iNbThread = cls.detectNumberOfCPUs() EDVerbose.DEBUG( "Initializing EDUtilsParallel semaphoreNbThreads to %s" % _iNbThread) cls._iNbThreads = _iNbThread cls._semaphoreNbThreads = Semaphore(_iNbThread) else: if cls._iNbThreads != _iNbThread: EDVerbose.WARNING( "cls._semaphoreNbThreads was already initialized to %s, not changing to %s" % (cls._iNbThreads, _iNbThread)) @classmethod def uninitializeNbThread(cls): """ For testing purpose: un-initialize the semaphore controlling the number of execPlugin running at once """ if cls._semaphoreNbThreads is not None: EDVerbose.DEBUG( "resetting the semaphore concerning the number of threads.") cls._semaphoreNbThreads = None @classmethod def getSemaphoreValue(cls): """ Class method: getter for the current value of the semaphore counting the CPU-active threads @return: value of the semaphore (or None if not initialized) @rtype: int """ if cls._semaphoreNbThreads is not None: return cls._semaphoreNbThreads._Semaphore__value else: return None @classmethod def getNbRunning(cls): """ Class method: getter for the number of CPU-active threads running @return: the number of CPU-active threads runnings @rtype: integer """ if cls._semaphoreNbThreads is not None: return cls._iNbThreads - cls._semaphoreNbThreads._Semaphore__value else: return None @classmethod def semaphoreNbThreadsAcquire(cls): """ Class method: Method to acquire the semaphore that controls the number of threads running concurrently """ if cls._semaphoreNbThreads is not None: cls._semaphoreNbThreads.acquire() else: EDVerbose.DEBUG( "EDUtilsParallel: Unable to acquire an uninitialized semaphore (NbCPU)." ) @classmethod def semaphoreNbThreadsRelease(cls): """ Class method: Method to release the semaphore that controls the number of threads running concurrently """ if cls._semaphoreNbThreads is not None: if cls._semaphoreNbThreads._Semaphore__value < cls._iNbThreads: cls._semaphoreNbThreads.release() else: EDVerbose.DEBUG( "EDUtilsParallel: Unable to release an uninitialized semaphore (NbCPU)." ) @classmethod def getSemaphoreNbThreads(cls): """ classmethod that returns the semaphore nb threads (to be used with the with_statement) @return: semaphore controling CPUs """ if cls._semaphoreNbThreads is not None: return cls._semaphoreNbThreads else: EDVerbose.DEBUG( "EDUtilsParallel: Unable to acquire an uninitialized semaphore (NbCPU)." )