def cleanDispMat(_path=None): """ This is to ensure that the displacement matrices are removed between 2 tests, if _path is given, try to backup file in this directory @param _path: path to backu file to @type _path: python string representing a path. """ for onefile in os.listdir( EDPluginSPDCorrectv10.getSPDCommonDirectory()): if onefile.endswith(".edf"): myFile = os.path.join( EDPluginSPDCorrectv10.getSPDCommonDirectory(), onefile) if _path is not None: try: shutil.move(myFile, os.path.join(_path, onefile)) except IOError: EDVerbose.WARNING( "Problems in moving EDF displacement matrices from %s to %s" % (myFile, _path)) else: try: os.remove(myFile) except OSError: EDVerbose.WARNING( "Problems in removing EDF displacement matrices: %s" % myFile)
def initSpec(_strSpecVersion, _strSpecStatus, _strSpecAbort): """ Initialization of SpecVariable ... """ if EDUtilsBioSaxs.specVersion is None: EDUtilsBioSaxs.__strSpecVersion = _strSpecVersion EDUtilsBioSaxs.__strSpecStatus = _strSpecStatus EDUtilsBioSaxs.__strSpecAbort = _strSpecAbort if SpecVariable: EDUtilsBioSaxs.__specVarStatus = SpecVariable.SpecVariable( _strSpecStatus) EDUtilsBioSaxs.__specVarAbort = SpecVariable.SpecVariable( _strSpecAbort) else: EDVerbose.DEBUG( "EDUtilsBioSaxs initSpec called whereas it was already set-up") if EDUtilsBioSaxs.__strSpecVersion != _strSpecVersion: EDVerbose.WARNING( "EDUtilsBioSaxs initSpec specVersion %s whereas configured with %s" % (_strSpecVersion, EDUtilsBioSaxs.__strSpecVersion)) if EDUtilsBioSaxs.__strSpecStatus != _strSpecStatus: EDVerbose.WARNING( "EDUtilsBioSaxs initSpec specStatus %s whereas configured with %s" % (_strSpecVersion, EDUtilsBioSaxs.__strSpecVersion)) if EDUtilsBioSaxs.__strSpecAbort != _strSpecAbort: EDVerbose.WARNING( "EDUtilsBioSaxs initSpec specAbort %s whereas configured with %s" % (_strSpecVersion, EDUtilsBioSaxs.__strSpecVersion))
def getEdnaUserTempFolder(cls): """ Returns the name of a temporary folder that is unique for a given user. """ if cls._EDNA_USERTEMPFOLDER is None: if os.environ.has_key("EDNA_USERTEMPFOLDER"): cls._EDNA_USERTEMPFOLDER = os.environ["EDNA_USERTEMPFOLDER"] if not os.path.exists(cls._EDNA_USERTEMPFOLDER): EDVerbose.warning( "EDNA_USERTEMPFOLDER environment variable is set to %s put the directory does not exist!" % cls._EDNA_USERTEMPFOLDER) cls._EDNA_USERTEMPFOLDER = None elif (not os.access(cls._EDNA_USERTEMPFOLDER, os.W_OK)): EDVerbose.warning( "EDNA_USERTEMPFOLDER environment variable is set to %s put the directory cannot be accessed!" % cls._EDNA_USERTEMPFOLDER) cls._EDNA_USERTEMPFOLDER = None if cls._EDNA_USERTEMPFOLDER is None: strEdnaTempFileDir = tempfile.gettempdir() try: # Working on Windows and Linux: strUserName = getpass.getuser() except Exception: # Working on MacOS: strUserName = os.getlogin() bIsOk = False # Check that we have write access to this directory: if os.access(strEdnaTempFileDir, os.W_OK) and os.access( strEdnaTempFileDir, os.X_OK): cls._EDNA_USERTEMPFOLDER = os.path.join( strEdnaTempFileDir, "edna-%s" % strUserName) # Check that we have write access to this directory: if not os.path.exists(cls._EDNA_USERTEMPFOLDER): try: os.mkdir(cls._EDNA_USERTEMPFOLDER) except Exception: EDVerbose.WARNING( "Error when trying to create the directory %s" % cls._EDNA_USERTEMPFOLDER) if os.access(cls._EDNA_USERTEMPFOLDER, os.W_OK) and os.access( cls._EDNA_USERTEMPFOLDER, os.X_OK): bIsOk = True if not bIsOk: # We cannot use the edna-<user name> folder... EDVerbose.WARNING( "EDUtilsFile.getEdnaUserTempFolder: cannot access user temporary directory %s" % cls._EDNA_USERTEMPFOLDER) # Create temporary directory cls._EDNA_USERTEMPFOLDER = tempfile.mkdtemp(prefix="edna-") EDVerbose.WARNING( "Created temporary directory for this session: %s" % cls._EDNA_USERTEMPFOLDER) EDVerbose.WARNING( "If you would like to continue to use this directory for future sessions" ) EDVerbose.WARNING( "please set then environment variable EDNA_USERTEMPFOLDER to %s" % cls._EDNA_USERTEMPFOLDER) return cls._EDNA_USERTEMPFOLDER
def unitTestExecute(self): """ check the execution of a job (without callback) """ EDVerbose.DEBUG("EDTestCaseEDJob.unitTestExecute") edJob = EDJob(self.strPluginName) strJobId = edJob.getJobId() edJob.setDataInput(self.strXmlInput) ref = edJob.execute() EDAssert.equal(strJobId, ref, "JobId has not changed") strStatus = edJob.getStatus() EDVerbose.WARNING("Job %s in State %s" % (strJobId, strStatus)) while strStatus in ["running", "uninitialized"]: EDVerbose.WARNING("Job %s in state %s" % (strJobId, strStatus)) time.sleep(0.01) strStatus = edJob.getStatus() xsdOut = edJob.getDataOutput() while xsdOut is None: EDVerbose.WARNING("No Output data, still waiting for output data to arrive, %s" % edJob.getStatus()) time.sleep(0.01) xsdOut = edJob.getDataOutput() strOutput = xsdOut.strip() strStatus = edJob.getStatus() while strStatus == "running": EDVerbose.WARNING("Job %s is still in state %s" % (strJobId, strStatus)) time.sleep(0.01) strStatus = edJob.getStatus() EDAssert.equal(strOutput, self.strXmlInput, "Output is OK") EDAssert.equal("success", edJob.getStatus(), "Job %s is finished with ''success''" % edJob.getJobId())
def execute(self): """ Launch the EDNA plugin @return: JobId @rtype: string """ returnId = None if not self.__bXmlInputSet: EDVerbose.WARNING("Not executing job %s as input is empty" % self.__jobId) if (self.__edPlugin is not None): self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_RUNNING self.__edPlugin.connectSUCCESS(self.successPluginExecution) self.__edPlugin.connectFAILURE(self.failurePluginExecution) self.__edPlugin.execute() returnId = self.__jobId self.synchronizeOff() else: EDVerbose.WARNING( "Trying to run a plugin that does not exist: %s " % self.__strPluginName) return returnId
def checkParameters(self): """ Checks the mandatory parameters """ EDVerbose.DEBUG("EDPluginControlCharacterisationv2_0.checkParameters") # Check for mxv1InputCharacterisation: if self.hasDataInput(): if self.hasDataInput("mxv1InputCharacterisation" ) or self.hasDataInput("mxv2DataCollection"): EDVerbose.WARNING( "Ambiguous input! Both XSDataInputCharacterisatiov2_0 input, and mxv1InputCharacterisation or mxv2DataCollection input(s), are given" ) self.setDataInput( self.getDataInput().getMxv1InputCharacterisation().marshal(), "mxv1InputCharacterisation") self.setDataInput( self.getDataInput().getMxv2DataCollection().marshal(), "mxv2DataCollection") # Check for mxv1InputCharacterisation self.checkMandatoryParameters( self.getDataInput("mxv1InputCharacterisation"), "Data Input is None") self.checkMandatoryParameters( self.getDataInput("mxv1InputCharacterisation") [0].getDataCollection(), "dataCollection") self.checkMandatoryParameters( self.getDataInput("mxv1InputCharacterisation") [0].getDataCollection().getDiffractionPlan(), "diffractionPlan")
def checkModeParameter(self): EDVerbose.DEBUG("EDPluginControlSolutionScatteringv0_2.checkModeParameter") try: if self.getDataInput().getMode().getValue().lower() in ['fast', 'slow']: self.__strMode = self.getDataInput().getMode().getValue().lower() except Exception: EDVerbose.WARNING("Running Solution Scattering pipeline in fast mode by default")
def createPyArchDNAFilePath(self, _strDNAFileDirectoryPath): """ This method translates from a "visitor" path to a "pyarch" path: /data/visitor/mx415/id14eh1/20100209 -> /data/pyarch/id14eh1/mx415/20100209 """ strPyarchDNAFilePath = None listOfDirectories = _strDNAFileDirectoryPath.split(os.sep) listBeamlines = ["bm14", "id14eh1", "id14eh2", "id14eh3", "id14eh4", "id23eh1", "id23eh2", "id29"] # Check that we have at least four levels of directories: if (len(listOfDirectories) > 4): strDataDirectory = listOfDirectories[ 1 ] strSecondDirectory = listOfDirectories[ 2 ] strProposal = None strBeamline = None if ((strDataDirectory == "data") and (strSecondDirectory == "visitor")): strProposal = listOfDirectories[ 3 ] strBeamline = listOfDirectories[ 4 ] elif ((strDataDirectory == "data") and (strSecondDirectory in listBeamlines)): strBeamline = strSecondDirectory strProposal = listOfDirectories[ 4 ] if (strProposal != None) and (strBeamline != None): strPyarchDNAFilePath = os.path.join(os.sep, "data") strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, "pyarch") strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, strBeamline) strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, strProposal) for strDirectory in listOfDirectories[ 5: ]: strPyarchDNAFilePath = os.path.join(strPyarchDNAFilePath, strDirectory) if (strPyarchDNAFilePath is None): EDVerbose.WARNING("EDPluginControlInterfaceToMXCuBEv1_2.createPyArchDNAFilePath: path not converted for pyarch: %s " % _strDNAFileDirectoryPath) return strPyarchDNAFilePath
def preImport(cls, _strModuleName, _strPath=None, _strForceVersion=None, _strMethodVersion=None): """ Static method that import locally with a lock and keeps track of already imported module. @param _strModuleName: Name of the module to import @param _strPath: Path to the module to import @param _strForceVersion: version string to enforce to. Should be compatible with the method given !!! @param _strMethodVersion: property or method to get the version number (should return a string) @return: reference to the module loaded """ oModule = None EDVerbose.DEBUG("EDFactoryPlugin.preImport %s %s %s is loaded=%s" % (_strModuleName, _strPath, _strForceVersion, _strModuleName in cls._dictLoadedModules)) if (_strModuleName not in cls._dictLoadedModules) or \ (cls._dictLoadedModules[_strModuleName].module is None): with cls.__semaphoreStatic: if _strModuleName not in cls._dictLoadedModules: edModule = EDModule(_strModuleName) cls._dictLoadedModules[_strModuleName] = edModule else: edModule = cls._dictLoadedModules[_strModuleName] oModule = edModule.preImport(_strPath, _strMethodVersion) elif (_strForceVersion is not None) and \ (cls._dictLoadedModules[_strModuleName].version < _strForceVersion): if (cls._dictLoadedModules[_strModuleName].version == "") and (_strMethodVersion is not None): cls._dictLoadedModules[_strModuleName].retrieveVersion(_strMethodVersion) if (cls._dictLoadedModules[_strModuleName].version < _strForceVersion): EDVerbose.WARNING("EDFactoryPlugin.preimport wrong module version: %s is %s not %s" % (_strModuleName, cls._dictLoadedModules[_strModuleName].version, _strForceVersion)) cls.unImport(_strModuleName) cls.preImport(_strModuleName, _strPath, _strForceVersion, _strMethodVersion) oModule = cls._dictLoadedModules[_strModuleName].module elif (cls._dictLoadedModules[_strModuleName].version == ""): cls._dictLoadedModules[_strModuleName].retrieveVersion(_strMethodVersion) oModule = cls._dictLoadedModules[_strModuleName].module else: oModule = cls._dictLoadedModules[_strModuleName].module return oModule
def checkDammifConstant(self): try: self.__strConstant = '--constant=' + str( self.getDataInput().getConstant().getValue()) except Exception: EDVerbose.WARNING( "Constant to subtract will be defined automatically")
def checkSupcombBackboneInput(self): EDVerbose.DEBUG("EDPluginExecSupcombv0_1.checkBackbone") try: if self.getDataInput().getBackbone().getValue(): self.__bBackbone = self.getDataInput().getBackbone().getValue() except Exception: EDVerbose.WARNING("Using all atoms in Supcomb by default")
def doFailureEvaluationIndexingLABELIT(self, _edPlugin=None): EDVerbose.DEBUG("EDPluginControlCharacterisationv1_2.doFailureEvaluationIndexing") strWarningMessage = "Execution of indexing evaluation plugin failed." EDVerbose.WARNING(strWarningMessage) self.addWarningMessage(strWarningMessage) if self.__xsDataResultCharacterisation is not None: self.setDataOutput(self.__xsDataResultCharacterisation)
def preProcess(self, _edObject=None): EDPluginExecProcessScript.preProcess(self) EDVerbose.DEBUG("EDPluginExecSaxsMacv1_0.preProcess") xsdIn = self.getDataInput() if xsdIn.getInputImage() is not None: self.inputImage = xsdIn.getInputImage().getPath().getValue() if not os.path.isfile(self.inputImage) and "%" not in self.inputImage: EDVerbose.WARNING("Input file %s does not exist ... try to go on anyway" % self.inputImage) self.inputImage = None if xsdIn.getOutputImage() is not None: self.outputImage = xsdIn.getOutputImage().getPath().getValue() if xsdIn.getFirstImage() is not None: self.firstImage = xsdIn.getFirstImage().getValue() if xsdIn.getLastImage() is not None: self.lastImage = xsdIn.getLastImage().getValue() if xsdIn.getOptions() is not None: self.options = xsdIn.getOptions().getValue() if xsdIn.getDummy() is not None: self.dummy = xsdIn.getDummy().getValue() if xsdIn.getIncrement() is not None: self.increment = xsdIn.getIncrement().getValue() if xsdIn.getAddConst() is not None: self.addConst = xsdIn.getAddConst().getValue() if xsdIn.getMultConst() is not None: self.multConst = xsdIn.getMultConst().getValue() #Create the command line to run the program self.generateSaxsMacCommand()
def copyHTMLDir(_strPathToHTMLDir, _strPathToPyarchDirectory): if not os.path.exists(_strPathToPyarchDirectory): try: os.mkdir(_strPathToPyarchDirectory) except: EDVerbose.WARNING( "EDHandlerESRFPyarchv1_0.copyHTMLFilesAndDir: cannot create pyarch html directory %s" % _strPathToPyarchDirectory) return elif not os.path.exists(_strPathToHTMLDir): EDVerbose.ERROR( "EDHandlerESRFPyarchv1_0.copyHTMLFilesAndDir: path to html directory does not exist: %s" % _strPathToHTMLDir) else: try: strPathToPyArchHtmlDirectory = os.path.join( _strPathToPyarchDirectory, "index") if os.path.exists(strPathToPyArchHtmlDirectory): shutil.rmtree(strPathToPyArchHtmlDirectory, ignore_errors=True) shutil.copytree(_strPathToHTMLDir, strPathToPyArchHtmlDirectory) except Exception, e: EDVerbose.ERROR( "EDHandlerESRFPyarchv1_0.copyHTMLFilesAndDir: Exception caught: %r" % e)
def __init__(self, _strPluginName): """ Constructor of the class @param strPluginName: name of the plugin @type strPluginName: string """ EDObject.__init__(self) self.__strPluginName = _strPluginName self.__edPlugin = None self.__edSlotCallBack = EDSlot() self.__edSlotSUCCESS = EDSlot() self.__edSlotFAILURE = EDSlot() self.__bXmlInputSet = False self.__status = None # self.__edPlugin = EDJob.__edFactoryPlugin.loadPlugin(self.__strPluginName) self.__edPlugin = EDPluginWrapperForJobScheduler(self.__strPluginName) EDJob.__semaphore.acquire() # Create the JobID if not self.__strPluginName in EDJob.__dictPluginLastId: EDJob.__dictPluginLastId[_strPluginName] = 0 else: EDJob.__dictPluginLastId[_strPluginName] += 1 self.__jobId = "%s-%i" % (self.__strPluginName, EDJob.__dictPluginLastId[_strPluginName]) EDJob.__dictJobs[self.__jobId] = self EDJob.__semaphore.release() if (self.__edPlugin is None): EDVerbose.WARNING("Instantiation of plugin %s failed!!!" % _strPluginName) else: self.__status = EDJob.PLUGIN_STATE_UNITIALIZED
def wrapper(*arg, **kw): '''This is the docstring from timeit: a decorator that prints the execution time''' t1 = time.time() res = func(*arg, **kw) EDVerbose.WARNING("%s took %.3fs" % (func.func_name, time.time() - t1)) return res
def addTestCaseFromName(self, _strTestCaseName): """ This method adds a test case give it's name to the test suite. If the test case cannot be loaded an error message is issued and it's name is added to the list of not executed test cases. @return: NumberTestCaseFailure @rtype: integer """ edTestCase = None exceptionObject = None try: edTestCase = EDUtilsTest.getFactoryPluginTest().loadPlugin( _strTestCaseName) except ImportError as exceptionObject: strWarningMessage = "Could not create the test case: %s, reason: %s" % ( _strTestCaseName, exceptionObject) EDVerbose.WARNING(strWarningMessage) self.__dictTestCaseNotExecuted[_strTestCaseName] = "%s : %s" % ( self.getClassName(), strWarningMessage) if edTestCase is None: if exceptionObject is None: EDVerbose.error( "EDTestSuite.addTestCaseFromName: Could not create the test case: " + _strTestCaseName) self.__dictTestCaseNotExecuted[ _strTestCaseName] = "%s : Could not create the test case" % self.getClassName( ) else: edTestCase.setTestSuiteName(self.getClassName()) self.__listTestCase.append(edTestCase)
def uploadReferenceFrame(self, entry): paths = entry.split(":") if len(paths) == 1 and os.path.isfile(paths[0]): obj = fabio.open(paths[0]).data EDVerbose.WARNING("Got reference frame %s via fabio" % entry) EDPluginControlAlignStackv1_0.addFrame(-1, obj) EDPluginAccumulatorv1_0.addItem(XSDataString("raw -001")) self.updateAttrRefImage(obj) elif len(paths) == 2 and os.path.isfile(paths[0]): hdf = h5py.File(paths[0]) if paths[1] in hdf: obj = hdf[paths[1]] if obj.__class__.__name__ == "Group": if self.REFERENCE_FRAME_NAME in obj: data = obj[self.REFERENCE_FRAME_NAME][:] EDPluginControlAlignStackv1_0.addFrame(-1, data) EDPluginAccumulatorv1_0.addItem("raw -001") self.updateAttrRefImage(data) else: EDVerbose.ERROR("HDF5: No '%s' in group %s from %s" % (self.REFERENCE_FRAME_NAME, paths[1], paths[0])) elif obj.__class__.__name__ == "Dataset": data = obj[:] EDPluginControlAlignStackv1_0.addFrame(-1, data) EDPluginAccumulatorv1_0.addItem(XSDataString("raw -001")) self.updateAttrRefImage(data) else: EDVerbose.ERROR("HDF5: No such internal path %s in %s" % (paths[1], paths[0])) else: EDVerbose.ERROR("No such entry %s" % entry)
def doFailureGeneratePrediction(self, _edPlugin=None): EDVerbose.DEBUG("EDPluginControlCharacterisationv1_2.doFailureGeneratePrediction") strWarningMessage = "Execution of generate prediction plugin failed." EDVerbose.WARNING(strWarningMessage) self.addWarningMessage(strWarningMessage) if self.__xsDataResultCharacterisation is not None: self.setDataOutput(self.__xsDataResultCharacterisation)
def checkSupcombEnantiomorphsInput(self): EDVerbose.DEBUG("EDPluginExecSupcombv0_1.checkEnantiomorphs") try: if self.getDataInput().getEnantiomorphs().getValue(): self.__bEnantiomorphs = self.getDataInput().getEnantiomorphs().getValue() except Exception: EDVerbose.WARNING("Enabling enantiomorphs in Supcomb by default")
def toXSD( _classXSData, _strObject, ): """Convert a string or possibly any object to an XSD object @param _classXSData: the XSDataClass to be forced into @type _classXSData: class derived from XSData @param _strObject: the string representation of a physical object like "1.54 A" @type _strObject: string (floats are accepted) """ xsd = _classXSData() if isinstance(_strObject, (unicode, str)): listWords = _strObject.split(None, 1) if len(listWords) > 0: try: fValue = float(listWords[0]) except Exception: EDVerbose.ERROR( "Trying to create XSData object from %s; fValue not a float !" % _strObject) else: xsd.setValue(fValue) if len(listWords) == 2: xsd.setUnit(XSDataString(listWords[1])) else: EDVerbose.WARNING("Trying to create XSData object from %s " % _strObject) elif isinstance(_strObject, (float, int)): xsd.setValue(_strObject) return xsd
def setDataInput(self, _oDataInput, _strDataInputKey=None): """ Sets the job (plugin) input data. @param: _oDataInput: could be either an String XML or an XSData object. @param _strDataInputKey: the key of an input data dictionnary The input data is stored in a dictionary with the key _strDataInputKey. If the key is not provided a default key is used. If not data input class is defined for the key an exception is raised. If the key is not the default key, the data object is added to a list which might contain already stored object(s). If _oDataInput is None the list corresponding to a keyword is deleted. """ if _oDataInput in ["", None]: self.__bXmlInputSet = False return self.synchronizeOn() if (self.__edPlugin is not None): self.__edPlugin.setDataInput(_oDataInput, _strDataInputKey) self.__bXmlInputSet = True else: EDVerbose.WARNING( "Setting DataInput for uninstanciated plugin %s." % self.__strPluginName) self.synchronizeOff()
def checkDamaverAutomaticInput(self): try: if self.getDataInput().getAutomatic(): self.__bAutomatic = self.getDataInput().getAutomatic( ).getValue() except Exception: EDVerbose.WARNING( "Running Damaver automation pipeline by default.")
def checkDammifParticleShapeInput(self): particleShape = ['PROLATE', 'OBLATE', 'UNKNOWN'] try: if self.getDataInput().getExpectedParticleShape().getValue( ) in range(3): self.__strParticleShape = particleShape[ self.getDataInput().getExpectedParticleShape().getValue()] except Exception: EDVerbose.WARNING("Using Unknown particle shape")
def wrapper(*arg, **kw): """ decorator that deprecates the use of a function """ EDVerbose.WARNING( "%s is Deprecated !!! %s" % (func.func_name, os.linesep.join([""] + traceback.format_stack()[:-1]))) return func(*arg, **kw)
def checkDamminMode(self): try: if self.getDataInput().getMode().getValue().upper() in [ 'FAST', 'SLOW' ]: self.__strMode = self.getDataInput().getMode().getValue( ).upper()[0] except Exception: EDVerbose.WARNING("Running DAMMIN in FAST mode by default")
def releaseFile(cls, filename): """ release the semaphore for this specific file """ if filename in cls.__dictLock: cls.__dictLock[filename].release() else: EDVerbose.WARNING( "EDPluginHDF5.releaseFile: file not under supervision %s " % filename)
def getDataOutput(self, _strDataOutputKey=None): """ Returns the Plugin Output Data """ if (self.__edPlugin is not None): return self.__edPlugin.getDataOutput(_strDataOutputKey) else: EDVerbose.WARNING( "Getting DataOutput for uninstanciated plugin %s." % self.__strPluginName)
def XMLerr(strXMLin): """ This is an example of XMLerr function ... it prints only the name of the file created @param srXMLin: The XML string used to launch the job @type strXMLin: python string with the input XML @rtype: None @return: None """ EDVerbose.WARNING("Error in the processing of :\n%s" % strXMLin) return None
def getFileLock(cls, filename): """ return the semaphore for locking .... """ if filename in cls.__dictLock: return cls.__dictLock[filename] else: EDVerbose.WARNING( "EDPluginHDF5.getFileSem: file not under supervision %s. Expect failure ! " % filename)