def run(self): """ Initialize and run all parallel jobs """ for __dictKey in self.__dictXSDataInput.keys(): self.__xsEDPluginExecJobs[ __dictKey] = self.__edControlPlugin.loadPlugin( self.__strPluginName) if (self.__xsEDPluginExecJobs[__dictKey] is not None): if (self.__dictXSDataInput[__dictKey] is not None) and (self.__dictXSDataInput[__dictKey] is not ""): self.__xsEDPluginExecJobs[__dictKey].setDataInput( self.__dictXSDataInput[__dictKey]) self.__xsEDPluginExecJobs[__dictKey].connectSUCCESS( self.__successPluginExecution) self.__xsEDPluginExecJobs[__dictKey].connectFAILURE( self.__failurePluginExecution) self.__xsEDActionCluster.addAction( self.__xsEDPluginExecJobs[__dictKey]) else: EDVerbose.screen("ERROR! Input data not found in " + self.__xsEDPluginExecJobs[__dictKey]. getWorkingDirectory()) else: EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName) self.__xsEDActionCluster.setTimeOut( self.__edControlPlugin.getTimeOut()) self.__xsEDActionCluster.executeSynchronous()
def runXAFSProcessing(): if request.GET.get('nxs', '').strip(): _dataPattern = request.GET.get('nxs', '').strip() _dataPattern = '/'.join(_dataPattern.split('\\\\')) print _dataPattern _strDatasetFileNames = [ os.path.abspath(tmp) for tmp in filter( lambda tmp: os.path.isfile(tmp), glob.glob(_dataPattern)) ] if (_strDatasetFileNames is not None): for _tmpName in _strDatasetFileNames: EDVerbose.screen("Reading input data from : %s" % _tmpName) else: return edApplicationXAFSBatchProcessing.usage() edApplicationXAFSBatchProcessing.setDatasetFileNames( _strDatasetFileNames) edApplicationXAFSBatchProcessing.execute() #return os.path.abspath(edApplicationXAFSBatchProcessing.getXAFSPlugin().getWorkingDirectory()) return os.path.join( os.path.abspath(edApplicationXAFSBatchProcessing.getXAFSPlugin(). getWorkingDirectory()), 'results.nxs') else: return edApplicationXAFSBatchProcessing.usage()
def doSuccessEvaluationIndexingMOSFLM(self, _edPlugin=None): EDVerbose.DEBUG("EDPluginControlCharacterisationv1_2.doSuccessEvaluationIndexingMOSFLM") self.retrieveSuccessMessages(_edPlugin, "EDPluginControlCharacterisationv1_2.doSuccessEvaluationIndexing") # Retrieve status messages (if any) if self.__edPluginExecEvaluationIndexingMOSFLM.hasDataOutput("statusMessageImageQualityIndicators"): self.addStatusMessage(self.__edPluginExecEvaluationIndexingMOSFLM.getDataOutput("statusMessageImageQualityIndicators")[0].getValue()) if self.__edPluginExecEvaluationIndexingMOSFLM.hasDataOutput("statusMessageIndexing"): self.addStatusMessage("MOSFLM: " + self.__edPluginExecEvaluationIndexingMOSFLM.getDataOutput("statusMessageIndexing")[0].getValue()) # Check if indexing was successful bIndexWithLabelit = False bIndexingSuccess = self.__edPluginExecEvaluationIndexingMOSFLM.getDataOutput("indexingSuccess")[0].getValue() if bIndexingSuccess: xsDataIndexingResult = self.__edPluginExecEvaluationIndexingMOSFLM.getDataOutput("indexingResult")[0] self.__xsDataIndexingResultMOSFLM = xsDataIndexingResult # Check if space group is P1 - if yes run Labelit indexing xsDataIndexingSolutionSelected = xsDataIndexingResult.getSelectedSolution() xsDataCrystal = xsDataIndexingSolutionSelected.getCrystal() xsDataSpaceGroup = xsDataCrystal.getSpaceGroup() strSpaceGroupName = xsDataSpaceGroup.getName().getValue().upper() # Check if MOSFLM has indexed in P1 if strSpaceGroupName == "P1": # Check if the user maybe asked for P1! bIndexWithLabelit = True if self.__xsDataCollection.getDiffractionPlan() is not None: if self.__xsDataCollection.getDiffractionPlan().getForcedSpaceGroup() is not None: if self.__xsDataCollection.getDiffractionPlan().getForcedSpaceGroup().getValue().upper() == "P1": EDVerbose.screen("P1 space forced by diffraction plan") bIndexWithLabelit = False if bIndexWithLabelit: EDVerbose.screen("P1 space group choosed - reindexing with Labelit") else: EDVerbose.screen("MOSFLM indexing successful!") if self.__edPluginControlIndexingIndicators.hasDataOutput("indexingShortSummary"): self.__strCharacterisationShortSummary += self.__edPluginControlIndexingIndicators.getDataOutput("indexingShortSummary")[0].getValue() # Generate prediction images xsDataCollection = self.__xsDataResultCharacterisation.getDataCollection() self.__xsDataResultCharacterisation.setIndexingResult(xsDataIndexingResult) xsDataGeneratePredictionInput = XSDataGeneratePredictionInput() xsDataGeneratePredictionInput.setDataCollection(XSDataCollection.parseString(xsDataCollection.marshal())) xsDataGeneratePredictionInput.setSelectedIndexingSolution(XSDataIndexingSolutionSelected.parseString(xsDataIndexingResult.getSelectedSolution().marshal())) self.__edPluginControlGeneratePrediction.setDataInput(xsDataGeneratePredictionInput) # Start the generation of prediction images - we synchronize in the post-process self.__edPluginControlGeneratePrediction.execute() # Then start the integration of the reference images self.indexingToIntegration() else: EDVerbose.screen("Indexing with MOSFLM failed!") bIndexWithLabelit = True if bIndexWithLabelit: # Execute Labelit indexing EDVerbose.screen("Now trying to index with Labelit - please be patient...") xsDataIndexingInput = XSDataIndexingInput() xsDataSubWedgeList = self.__xsDataCollection.getSubWedge() xsDataExperimentalCondition = xsDataSubWedgeList[0].getExperimentalCondition() xsDataIndexingInput.setDataCollection(self.__xsDataCollection) xsDataIndexingInput.setExperimentalCondition(xsDataExperimentalCondition) if self.__xsDataCrystal != None: xsDataIndexingInput.setCrystal(self.__xsDataCrystal) self.__edPluginControlIndexingLabelit.setDataInput(xsDataIndexingInput) self.__edPluginControlIndexingLabelit.executeSynchronous()
def testExecute(self): """ """ plugin = self.getPlugin() plugin.__class__.dictHPLC = {} self.run() ################################################################################ # Compare XSDataResults ################################################################################ strExpectedOutput = self.readAndParseFile (self.getReferenceDataOutputFile()) EDVerbose.DEBUG("Checking obtained result...") xsDataResultReference = XSDataResultBioSaxsHPLCv1_0.parseString(strExpectedOutput) xsDataResultObtained = plugin.getDataOutput() EDAssert.strAlmostEqual(xsDataResultReference.marshal(), xsDataResultObtained.marshal(), "XSDataResult output are the same", _strExcluded="bioSaxs") ################################################################################ # Compare Ascii files ################################################################################ asciiObt = os.linesep.join([i.strip() for i in open(xsDataResultObtained.integratedCurve.path.value) if "Raster" not in i]) asciiRef = os.linesep.join([i.strip() for i in open(os.path.join(self.getTestsDataImagesHome(), "bioSaxsProcessIntegrated1_2.dat")) if "Raster" not in i]) EDAssert.strAlmostEqual(asciiRef, asciiObt, _strComment="3 column ascii files are the same", _fRelError=0.1, _strExcluded=os.environ["USER"]) EDVerbose.screen("Execution time for %s: %.3fs" % (plugin.getClassName(), plugin.getRunTime()))
def preProcess(self, _edObject=None): EDActionCluster.preProcess(self) EDVerbose.DEBUG("EDParallelJobLauncher.preProcess") for __dictKey in self.__dictXSDataInput.keys(): self.__xsEDPluginExecJobs[ __dictKey] = self.__edControlPlugin.loadPlugin( self.__strPluginName) if (self.__xsEDPluginExecJobs[__dictKey] is not None): if (self.__dictXSDataInput[__dictKey] is not None) and (self.__dictXSDataInput[__dictKey] is not ""): self.__xsEDPluginExecJobs[__dictKey].setDataInput( self.__dictXSDataInput[__dictKey]) self.addAction(self.__xsEDPluginExecJobs[__dictKey]) else: EDVerbose.screen("ERROR! Input data not found in " + self.__xsEDPluginExecJobs[__dictKey]. getWorkingDirectory()) else: EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName) self.setTimeOut(self.__edControlPlugin.getTimeOut())
def functionXMLin(_strFilename): """Here we create the XML string to be passed to the EDNA plugin from the input strFilename This can / should be modified by the final user @param _strFilename: full path of the input file @type _strFilename: python string representing the path @return: string """ EDVerbose.screen("Starting processing of image %s" % (_strFilename)) # First check if the filename end with .img or .mccd: strXML = None if (_strFilename.endswith(".img") or _strFilename.endswith(".mccd") or _strFilename.endswith(".cbf")): xsDataInputGridScreening = XSDataInputGridScreening() xsDataDiffractionPlan = XSDataDiffractionPlan() xsDataDiffractionPlan.setMaxExposureTimePerDataCollection(XSDataTime(EDParallelExecuteGridScreening.fMaxExposureTime)) xsDataInputGridScreening.setDiffractionPlan(xsDataDiffractionPlan) xsDataFile = XSDataFile() xsDataFile.setPath(XSDataString(_strFilename)) xsDataInputGridScreening.setImageFile(xsDataFile) if EDParallelExecuteGridScreening.bOnlyImageQualityIndicators: xsDataInputGridScreening.setDoOnlyImageQualityIndicators(XSDataBoolean(True)) if EDParallelExecuteGridScreening.bStoreInISPyB: xsDataInputGridScreening.setStoreImageQualityIndicatorsInISPyB(XSDataBoolean(True)) if EDParallelExecuteGridScreening.bDoOnlyIntegrationWithXMLOutput: xsDataInputGridScreening.setDoOnlyIntegrationWithXMLOutput(XSDataBoolean(True)) strXML = xsDataInputGridScreening.marshal() else: EDVerbose.screen("File name not ending with .img or .mccd - ignored : %s" % _strFilename) return strXML
def showData(cls): EDVerbose.screen("*" * 20 + "EDPluginControlAlignStackv1_0" + "*" * 20) EDVerbose.screen("Reference Frame: %s" % cls.__iRefFrame) if len(cls.__dictRelShift) < len(cls.__dictAbsShift): mydict = cls.__dictAbsShift.copy() else: mydict = cls.__dictRelShift.copy() lstTxt = [] for i in mydict: txt = "Frame %4i:\t" % i rela = cls.__dictRelShift.get(i) abso = cls.__dictAbsShift.get(i) if rela: txt += "relative: (%.3f, %.3f)\t" % rela else: txt += "relative: %12s\t" % rela if abso: txt += "absolute: (%.3f, %.3f)" % abso else: txt += "absolute: %12s" % abso lstTxt.append(txt) EDVerbose.screen(os.linesep.join(lstTxt)) items = EDPluginAccumulatorv1_0.getItems() items.sort() EDVerbose.screen("Items in the accumulator: %s" % (items)) querylist = [" "] + [ str(i) for i in EDPluginAccumulatorv1_0.getQueries().keys() ] EDVerbose.screen("Queries in the accumulator: " + os.linesep.join(querylist))
def start(self, _strXmlInput): """ Launch EDNA with the given XML stream @param _strXmlInput: XML to be passed to the plugin @type _strXmlInput: python string representing the XML data structure """ if (_strXmlInput is None) or (_strXmlInput == "") : return #This is a trick to work-around bug #463: # Run the fist thread alone (delay the second, third, ...) # semaphore._Semaphore__value is the current value of the value, unfortunatly it is a protected value without getter # I need the value of the semaphore to guess if the current call is the first or not. # Nota semaphore are decreased in value from self.__iNbThreads to 0. When Zero, the semaphore is blocking. # Them all other, limited by the semaphore. if self.__bIsFirstExecute: sys.stdout.write("Waiting for first thread to initialize ....") while EDUtilsParallel.getNbRunning() > 0: time.sleep(0.5) sys.stdout.write(".") EDUtilsParallel.semaphoreNbThreadsAcquire() edPlugin = EDParallelExecute.__edFactoryPlugin.loadPlugin(self.__strPluginName) if (edPlugin is not None): edPlugin.setDataInput(_strXmlInput) edPlugin.connectSUCCESS(self.successPluginExecution) edPlugin.connectFAILURE(self.failurePluginExecution) edPlugin.execute() #edPlugin.executeSynchronous() else: EDUtilsParallel.semaphoreNbThreadsRelease() self.__bIsFirstExecute = False EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName)
def runEdnaFunction(self, _listNewFiles, _bIncludeSubdirs=False): """ This method is the launcher for new files found by watch_directories ; it is also called directly in offline mode. @param _listNewFiles: list of files newly created in the directory. @type _listNewFiles: python list of strings. @param _bIncludeSubdirs: should we include sub-directories ? yes for offline and no for online. @type _bIncludeSubdirs: boolean """ dictListFiles = {} for oneFile in _listNewFiles: if os.path.isfile(oneFile): if self.__bQuit == True: return base = list(os.path.splitext(oneFile)[0]) base.reverse() for i in xrange(len(base)): if not base[i].isdigit(): break base = base[i:] base.reverse() base = "".join(base) if base in dictListFiles: dictListFiles[base].append(oneFile) else: dictListFiles[base] = [oneFile] for key in dictListFiles: EDVerbose.screen("Processing %s: %s" % (key, dictListFiles[key])) self.start(self.__functXMLin(dictListFiles[key]))
def flush(self): """ This method calls the functXMLin a few times with a flush=True argument or without arguments and finishes the work """ bFinished = False while not bFinished: xml = None try: xml = self.__functXMLin(None, flush=True) except TypeError: try: xml = self.__functXMLin("", flush=True) except TypeError: try: xml = self.__functXMLin("") except TypeError: try: xml = self.__functXMLin("") except TypeError: xml = None if (xml is None) or (xml == ""): bFinished = True else: EDVerbose.screen("Flushing data ...") self.start(xml)
def doSuccessEvaluationIndexingLABELIT(self, _edPlugin=None): EDVerbose.DEBUG( "EDPluginControlCharacterisationv1_3.doSuccessEvaluationIndexingLABELIT" ) self.retrieveSuccessMessages( _edPlugin, "EDPluginControlCharacterisationv1_3.doSuccessEvaluationIndexingLABELIT" ) # Retrieve status messages (if any) if self._edPluginExecEvaluationIndexingLABELIT.hasDataOutput( "statusMessageIndexing"): self.addStatusMessage( "Labelit: " + self._edPluginExecEvaluationIndexingLABELIT.getDataOutput( "statusMessageIndexing")[0].getValue()) # Check if indexing was successful bIndexingSuccess = self._edPluginExecEvaluationIndexingLABELIT.getDataOutput( "indexingSuccess")[0].getValue() if bIndexingSuccess: xsDataIndexingResult = self._edPluginExecEvaluationIndexingLABELIT.getDataOutput( "indexingResult")[0] self._xsDataResultCharacterisation.setIndexingResult( xsDataIndexingResult) # Then start the integration of the reference images self.indexingToIntegration() else: EDVerbose.screen("Indexing with LABELIT failed!") self.setFailure()
def showData(cls): EDVerbose.screen("*"*20 + "EDPluginControlAlignStackv1_0" + "*" * 20) EDVerbose.screen("Reference Frame: %s" % cls.__iRefFrame) if len(cls.__dictRelShift) < len(cls.__dictAbsShift): mydict = cls.__dictAbsShift.copy() else: mydict = cls.__dictRelShift.copy() lstTxt = [] for i in mydict: txt = "Frame %4i:\t" % i rela = cls.__dictRelShift.get(i) abso = cls.__dictAbsShift.get(i) if rela: txt += "relative: (%.3f, %.3f)\t" % rela else: txt += "relative: %12s\t" % rela if abso: txt += "absolute: (%.3f, %.3f)" % abso else: txt += "absolute: %12s" % abso lstTxt.append(txt) EDVerbose.screen(os.linesep.join(lstTxt)) items = EDPluginAccumulatorv1_0.getItems() items.sort() EDVerbose.screen("Items in the accumulator: %s" % (items)) querylist = [" "] + [ str(i) for i in EDPluginAccumulatorv1_0.getQueries().keys()] EDVerbose.screen("Queries in the accumulator: " + os.linesep.join(querylist))
def testExecute(self): """ """ self.run() plugin = self.getPlugin() ################################################################################ # Compare XSDataResults ################################################################################ strExpectedOutput = self.readAndParseFile (self.getReferenceDataOutputFile()) EDVerbose.DEBUG("Checking obtained result...") xsDataResultReference = XSDataResultBioSaxsProcessOneFilev1_0.parseString(strExpectedOutput) xsDataResultObtained = plugin.getDataOutput() EDAssert.strAlmostEqual(xsDataResultReference.marshal(), xsDataResultObtained.marshal(), "XSDataResult output are the same", _strExcluded="bioSaxs") ################################################################################ # Compare Ascii HEADER files ################################################################################ asciiObt = os.linesep.join([i.strip() for i in open(xsDataResultObtained.integratedCurve.path.value) if i.startswith("#") and "Raster" not in i]) asciiRef = os.linesep.join([i.strip() for i in open(os.path.join(self.getTestsDataImagesHome(), "bioSaxsProcessIntegrated1_2.dat")) if i.startswith("#") and "Raster" not in i]) EDAssert.strAlmostEqual(asciiObt, asciiRef, _strComment="ascii header files are the same", _fRelError=0.1, _strExcluded=os.environ["USER"]) dataObt = numpy.loadtxt(xsDataResultObtained.integratedCurve.path.value) dataRef = numpy.loadtxt(os.path.join(self.getTestsDataImagesHome(), "bioSaxsProcessIntegrated1_2.dat")) EDAssert.curveSimilar((dataObt[:, 0], dataObt[:, 1]), (dataRef[:, 0], dataRef[:, 1]), "data are the same", 0.6) EDAssert.curveSimilar((dataObt[:, 0], dataObt[:, 2]), (dataRef[:, 0], dataRef[:, 2]), "errors are the same", 0.6) EDVerbose.screen("Execution time for %s: %.3fs" % (plugin.getClassName(), plugin.getRunTime()))
def flush(self): """ This method calls the functXMLin a few times with a flush=True argument or without arguments and finishes the work """ bFinished = False while not bFinished: xml = None try: xml = self.__functXMLin(None, flush=True) except TypeError: try: xml = self.__functXMLin("", flush=True) except TypeError: try: xml = self.__functXMLin("") except TypeError: try: xml = self.__functXMLin("") except TypeError: xml = None if (xml is None) or (xml == ""): bFinished = True else: EDVerbose.screen ("Flushing data ...") self.start(xml)
def runEdnaFunction(self, _listNewFiles, _bIncludeSubdirs=False): """ This method is the launcher for new files found by watch_directories ; it is also called directly in offline mode. @param _listNewFiles: list of files newly created in the directory. @type _listNewFiles: python list of strings. @param _bIncludeSubdirs: should we include sub-directories ? yes for offline and no for online. @type _bIncludeSubdirs: boolean """ dictListFiles = {} for oneFile in _listNewFiles: if os.path.isfile(oneFile): if self.__bQuit == True: return base = list(os.path.splitext(oneFile)[0]) base.reverse() for i in xrange(len(base)): if not base[i].isdigit(): break base = base[i:] base.reverse() base = "".join(base) if base in dictListFiles: dictListFiles[base].append(oneFile) else: dictListFiles[base] = [oneFile] for key in dictListFiles: EDVerbose.screen ("Processing %s: %s" % (key, dictListFiles[key])) self.start(self.__functXMLin(dictListFiles[key]))
def functionXMLout(_strXMLin, _strXMLout): """ This is an example of XMLout function ... it prints only the name of the file created @param _srXMLin: The XML string used to launch the job @type _strXMLin: python string with the input XML @param _strXMLout: The XML string retrieved job @type _strXMLout: python string with the output XML @return: None """ xsDataInputInterface = XSDataInputInterface.parseString(_strXMLin) strImagePath = xsDataInputInterface.getImagePath()[0].getPath().getValue() EDVerbose.screen("Successful processing of image %s" % strImagePath) strResultText = os.path.dirname(strImagePath) + " " + os.path.basename(strImagePath) + " " xsDataResultInterface = XSDataResultInterface.parseString(_strXMLout) xsDataResultCharacterisation = xsDataResultInterface.getResultCharacterisation() # xsDataIndexingResult = xsDataResultCharacterisation.getIndexingResult() #strResultText = _strDirectoryVisit + " " + strFileName + " " if (xsDataIndexingResult is None): strResultText += "Indexing failed" else: xsDataSelectedSolution = xsDataIndexingResult.getSelectedSolution() fMosaicityEstimation = xsDataSelectedSolution.getCrystal().getMosaicity().getValue() strResultText += "Mosaicity: %.2f " % fMosaicityEstimation xsDataStrategyResult = xsDataResultCharacterisation.getStrategyResult() if (xsDataStrategyResult is None): strResultText += "No strategy result" else: xsDataCollectionPlan = xsDataStrategyResult.getCollectionPlan()[0] xsDataStrategySummary = xsDataCollectionPlan.getStrategySummary() fRankingResolution = xsDataStrategySummary.getRankingResolution().getValue() strResultText += "Resolution: %.2f" % fRankingResolution writeToResultFile(strResultText)
def doSuccessActionPlugin(self, _edPlugin): """ """ EDVerbose.DEBUG("EDApplication.doSuccessActionPlugin") # Print the potential Warnings and Errors self.__listWarningMessages = _edPlugin.getListOfWarningMessages() EDVerbose.DEBUG( "EDApplication.doSuccessActionPlugin: Plugin %s Successful with : %i Warnings " % (_edPlugin.getPluginName(), len(self.__listWarningMessages))) for warningMessage in self.__listWarningMessages: EDVerbose.screen(warningMessage) self.__listErrorMessages = _edPlugin.getListOfErrorMessages() EDVerbose.DEBUG( "EDApplication.doSuccessActionPlugin: Plugin %s Successful with : %i Errors" % (_edPlugin.getPluginName(), len(self.__listErrorMessages))) for errorMessage in self.__listErrorMessages: EDVerbose.error(errorMessage) if (_edPlugin.hasDataOutput()): xsDataOutput = _edPlugin.getDataOutput() if (xsDataOutput is not None and self.__strDataOutputFilePath is not None): xsDataOutput.exportToFile(self.__strDataOutputFilePath) if (xsDataOutput is not None and self.__edObtainedOutputDataFile is not None): xsDataOutput.exportToFile(self.__edObtainedOutputDataFile)
def processCommandLineVersion(self): EDVerbose.DEBUG("EDApplication.processCommandLineVersion") if (self.__edCommandLine.existCommand(EDApplication.VERSION_PARAM_LABEL_1) or self.__edCommandLine.existCommand(EDApplication.VERSION_PARAM_LABEL_2)): EDVerbose.setVerboseOn() EDVerbose.screen("%s version %s" % (self.__strName, self.__strVersion)) self.__bIsFailure = True
def functionXMLin(_strFilename): """Here we create the XML string to be passed to the EDNA plugin from the input strFilename This can / should be modified by the final user @param _strFilename: full path of the input file @type _strFilename: python string representing the path @return: string """ EDVerbose.screen("Starting processing of image %s" % (_strFilename)) # First check if the filename end with .img or .mccd: strXML = None if _strFilename.endswith(".img") or _strFilename.endswith(".mccd") or _strFilename.endswith(".cbf"): xsDataInputGridScreening = XSDataInputGridScreening() xsDataDiffractionPlan = XSDataDiffractionPlan() xsDataDiffractionPlan.setMaxExposureTimePerDataCollection( XSDataTime(EDParallelExecuteGridScreening.fMaxExposureTime) ) xsDataInputGridScreening.setDiffractionPlan(xsDataDiffractionPlan) xsDataFile = XSDataFile() xsDataFile.setPath(XSDataString(_strFilename)) xsDataInputGridScreening.setImageFile(xsDataFile) if EDParallelExecuteGridScreening.bOnlyImageQualityIndicators: xsDataInputGridScreening.setDoOnlyImageQualityIndicators(XSDataBoolean(True)) if EDParallelExecuteGridScreening.bStoreInISPyB: xsDataInputGridScreening.setStoreImageQualityIndicatorsInISPyB(XSDataBoolean(True)) if EDParallelExecuteGridScreening.bDoOnlyIntegrationWithXMLOutput: xsDataInputGridScreening.setDoOnlyIntegrationWithXMLOutput(XSDataBoolean(True)) strXML = xsDataInputGridScreening.marshal() else: EDVerbose.screen("File name not ending with .img or .mccd - ignored : %s" % _strFilename) return strXML
def generateIndicatorsShortSummary(self, _edPlugin): """ Generates a very short summary of the image quality indicator processing """ strIndicatorsShortSummary = "" if _edPlugin.hasDataOutput(): for xsDataQualityIndicators in _edPlugin.getDataOutput().getImageQualityIndicators(): strImageName = os.path.basename(xsDataQualityIndicators.getImage().getPath().getValue()) strIndicatorsShortSummary += "ImageQualityIndicators: %s: " % strImageName iNoGoodBraggCandidates = xsDataQualityIndicators.getGoodBraggCandidates().getValue() strIndicatorsShortSummary += "good bragg %d, " % iNoGoodBraggCandidates fResMethod1 = xsDataQualityIndicators.getMethod1Res().getValue() strIndicatorsShortSummary += "r1 %.1f [A], " % fResMethod1 if xsDataQualityIndicators.getMethod2Res() is not None: fResMethod2 = xsDataQualityIndicators.getMethod2Res().getValue() strIndicatorsShortSummary += "r2 %.1f [A], " % fResMethod2 if xsDataQualityIndicators.getMaxUnitCell() is not None: fMaxCell = xsDataQualityIndicators.getMaxUnitCell().getValue() strIndicatorsShortSummary += "max cell %.1f [A], " % fMaxCell iIceRings = xsDataQualityIndicators.getIceRings().getValue() strIndicatorsShortSummary += "ice rings %d" % iIceRings if xsDataQualityIndicators.getTotalIntegratedSignal() is not None: fTotalIntegratedSignal = xsDataQualityIndicators.getTotalIntegratedSignal().getValue() strIndicatorsShortSummary += ", total integrated signal %.0f\n" % fTotalIntegratedSignal for strLine in strIndicatorsShortSummary.split("\n"): EDVerbose.screen(strLine) self.setDataOutput(XSDataString(strIndicatorsShortSummary), "indicatorsShortSummary")
def moveToTempDir(self): """ Create a temporary directory and put all logs there """ self.__strCurrWorkDir = os.getcwd() self.__strTempDir = tempfile.mkdtemp(suffix='.log', prefix='edna-') EDVerbose.screen("The log directory of EDNA will be in " + self.__strTempDir) os.chdir(self.__strTempDir)
def processKernel(self): """ Executes the test case. """ EDVerbose.DEBUG("EDTestCase.processKernel") EDVerbose.screen() EDVerbose.unitTest( "===================================================================" ) if self.getTestSuiteName() is not None: EDVerbose.unitTest("TEST SUITE : %s" % self.getTestSuiteName()) EDVerbose.unitTest("TEST CASE : %s" % self.getClassName()) EDVerbose.unitTest(" ") if self.__strReasonForNotBeingExecuted == "": self.setTimeInit() iNumberMethods = self.getNumberOfTests() EDVerbose.unitTest("Total number of tests : %d" % iNumberMethods) EDVerbose.unitTest() iTestCaseNumber = 0 if self.getListTest() == []: self.__strReasonForNotBeingExecuted = "No test methods!" else: for pyTestMethod in self.getListTest(): iTestCaseNumber = iTestCaseNumber + 1 strMethodName = EDUtilsTest.patchMethodName(pyTestMethod) EDVerbose.unitTest( "-------------------------------------------------------------------" ) EDVerbose.unitTest("Test case method : %s" % strMethodName) EDVerbose.unitTest() try: pyTestMethod() self.__iNumberTestMethodSuccess += 1 EDVerbose.unitTest("%s executed with SUCCESS" % strMethodName) EDVerbose.unitTest() except AssertionError as pyException: self.__dictMethodFailureMessages[strMethodName] = str( pyException) self.__iNumberTestMethodFailure += 1 EDVerbose.unitTest("Assertion Error Raised!") EDVerbose.unitTest("%s executed with FAILURE" % strMethodName) EDVerbose.unitTest() except Exception as pyException: self.__dictMethodFailureMessages[strMethodName] = str( pyException) self.__iNumberTestMethodFailure += 1 EDVerbose.unitTest("Unexpected Error!") EDVerbose.unitTest("%s executed with FAILURE" % strMethodName) EDVerbose.unitTest() EDVerbose.writeErrorTrace() EDVerbose.unitTest() self.__bIsExecuted = True self.setTimeEnd()
def stats(cls): """ Retrieve some statistics and print them """ lstStrOut = [] output = [] fExecTime = time.time() - cls.__fStartTime keys = cls.__dictJobs.keys() keys.sort() for num, key in enumerate(keys): job = cls.__dictJobs[key] if job.getPlugin() is None: runtime = job.__runtime else: runtime = job.getPlugin().getRunTime() output.append( [num, key, job.getStatus(), runtime, job.getMemSize()]) output.sort() iNbJob = max(1, len(keys)) lstStrOut.append("_" * 110) lstStrOut.append( "%s\t|\t%s\t\t\t\t|\t%s\t|\t%s\t\t|\t%s" % ("nr", "EDPluginName-Id", "status", "runtime", "memory")) lstStrOut.append("_" * 110) fWall = 0.0 fSumProd = 0.0 fSumX = 0.0 fSumX2 = 0.0 for oneJob in output: fWall += oneJob[3] fSumX += oneJob[0] fSumX2 += oneJob[0] * oneJob[0] fSumProd += oneJob[0] * oneJob[3] lstStrOut.append("%s\t|\t%s\t|\t%s\t|\t%9.3f\t|\t%s" % tuple(oneJob)) lstStrOut.append("_" * 110) lstStrOut.append( "Total execution time (Wall): %.3fs, Execution time: %.3fs. SpeedUp: %.3f" % (fWall, fExecTime, fWall / fExecTime)) lstStrOut.append( "Average execution time (Wall/N): %.3fs, Average throughput: %.3fs" % (fWall / iNbJob, fExecTime / iNbJob)) if len(keys) > 1: fSlope = (iNbJob * fSumProd - fSumX * fWall) / (iNbJob * fSumX2 - fSumX * fSumX) fOrd = (fWall - fSlope * fSumX) / iNbJob else: fSlope = 0.0 fOrd = fWall lstStrOut.append( "Regression of execution time: ExecTime = %.3f + %f * NbJob" % (fOrd, fSlope)) strOutput = os.linesep.join(lstStrOut) EDVerbose.screen(strOutput) return strOutput
def testExecute(self): """ """ self.run() plugin = self.getPlugin() ################################################################################ # Compare XSDataResults ################################################################################ strExpectedOutput = self.readAndParseFile (self.getReferenceDataOutputFile()) EDVerbose.DEBUG("Checking obtained result...") xsDataResultReference = XSDataResultBioSaxsProcessOneFilev1_0.parseString(strExpectedOutput) xsDataResultObtained = plugin.getDataOutput() EDAssert.strAlmostEqual(xsDataResultReference.marshal(), xsDataResultObtained.marshal(), "XSDataResult output are the same", _strExcluded="bioSaxs") ################################################################################ # Compare dictionary ################################################################################ obtFile = xsDataResultObtained.integratedImage.path.value refFile = os.path.join(self.getTestsDataImagesHome(), "bioSaxsProcessIntegrated.edf") EDVerbose.DEBUG("fileRef=%s fileObt=%s" % (refFile, obtFile)) edfRef = openimage(refFile) edfObt = openimage(obtFile) headerRef = edfRef.header headerObt = edfObt.header keysRef = headerRef.keys() keysObt = headerObt.keys() keysRef.sort() keysObt.sort() # EDAssert.equal(keysRef, keysObt, _strComment="Same keys in the header dict") for key in keysRef: if not (key.startswith("History") or key in [ "Compression", "DDummy", "Dummy", "EDF_HeaderSize", "HeaderID", "Image", "filename", "RasterOrientation"]): EDAssert.strAlmostEqual(headerRef[key], headerObt[key], _strComment="header value %s are the same" % key, _strExcluded="bioSaxs") ################################################################################ # Compare images ################################################################################ outputData = edfRef.data referenceData = edfObt.data EDAssert.arraySimilar(numpy.maximum(outputData, 0), numpy.maximum(referenceData, 0) , _fScaledMaxDelta=0.05, _strComment="Images-data are the same") outputData = edfRef.next().data referenceData = edfObt.next().data EDAssert.arraySimilar(numpy.maximum(outputData, 0), numpy.maximum(referenceData, 0) , _fScaledMaxDelta=0.05, _strComment="Images-ESD are the same") ################################################################################ # Compare Ascii files ################################################################################ asciiObt = os.linesep.join([i.strip() for i in open(xsDataResultObtained.integratedCurve.path.value) if "Raster" not in i]) asciiRef = os.linesep.join([i.strip() for i in open(os.path.join(self.getTestsDataImagesHome(), "bioSaxsProcessIntegrated1_1.dat")) if "Raster" not in i]) EDAssert.strAlmostEqual(asciiRef, asciiObt, _strComment="3 column ascii files are the same", _fRelError=0.1, _strExcluded=os.environ["USER"]) EDVerbose.screen("Execution time for %s: %.3fs" % (plugin.getClassName(), plugin.getRunTime()))
def doSubWedgeAssembleFAILURE(self, _edPlugin): EDVerbose.DEBUG( "EDPluginControlInterfacev2_0.doSubWedgeAssembleFAILURE") EDVerbose.screen("Execution of " + self.strEDPluginControlSubWedgeAssembleName + " failed.") EDVerbose.screen( "Please inspect the log file for further information.") self.setFailure()
def getSPDCommonDirectory(): """ Create a temporary directory common for all SPD process / plugins in order to store displacement matrices @return: the name of the common directory @rtype: string """ if EDPluginSPDCorrectv10.__strSPDDir is None: EDPluginSPDCorrectv10.__strSPDDir = tempfile.mkdtemp(suffix=".tmp", prefix="edna-SPD-") EDVerbose.screen("The SPD-EDNA logs and matrices will be in " + EDPluginSPDCorrectv10.__strSPDDir) return EDPluginSPDCorrectv10.__strSPDDir
def getSPDCommonDirectory(): """ Create a temporary directory common for all SPD process / plugins in order to store displacement matrices @return: the name of the common directory @rtype: string """ if EDPluginSPDCorrectv10.__strSPDDir is None: EDPluginSPDCorrectv10.__strSPDDir = tempfile.mkdtemp(suffix='.tmp', prefix='edna-SPD-') EDVerbose.screen("The SPD-EDNA logs and matrices will be in " + EDPluginSPDCorrectv10.__strSPDDir) return EDPluginSPDCorrectv10.__strSPDDir
def doFailureActionPlugin(self, _edPlugin=None): """ retrieve the potential warning messages retrieve the potential error messages """ EDApplication.doFailureActionPlugin(self, _edPlugin) EDVerbose.DEBUG("EDApplicationXAFSBatchProcessing.doFailureActionPlugin") EDVerbose.screen("Execution of " + _edPlugin.getPluginName() + " failed.") EDVerbose.screen("Please inspect the log file for further information.")
def postProcess(self): """ This method writes out the results of the test suite. """ EDVerbose.screen() EDVerbose.screen() EDVerbose.unitTest("###################################################################") strResult = None if self.__iNumberTestCaseFailure == 0 and self.__iNumberTestMethodFailure == 0: strResult = "SUCCESS" else: strResult = "FAILURE" EDVerbose.unitTest("Result for %s : %s" % (self.getTestName(), strResult)) EDVerbose.unitTest() if self.__iNumberTestSuite > 0: EDVerbose.unitTest(" Number of executed test suites in this test suite : %d" % self.__iNumberTestSuite) EDVerbose.unitTest() if self.__dictTestSuiteNotExecuted != {}: EDVerbose.unitTest() EDVerbose.unitTest("OBS! The following test suites were not executed due to errors:") EDVerbose.unitTest() for strTestSuiteName in self.__dictTestSuiteNotExecuted.keys(): EDVerbose.unitTest(" %s : %s" % (self.__dictTestSuiteNotExecuted[strTestSuiteName], strTestSuiteName)) EDVerbose.unitTest() if self.__dictTestCaseNotExecuted != {}: EDVerbose.unitTest() EDVerbose.unitTest("OBS! The following test cases not executed due to errors:") EDVerbose.unitTest() for strTestCaseName in self.__dictTestCaseNotExecuted.keys(): EDVerbose.unitTest(" %s :" % strTestCaseName) EDVerbose.unitTest(" %s" % self.__dictTestCaseNotExecuted[strTestCaseName]) EDVerbose.unitTest() EDVerbose.unitTest(" Total number of test cases NOT EXECUTED : %d" % len(self.__dictTestCaseNotExecuted)) EDVerbose.unitTest() if self.__iNumberTestCaseSuccess != 0 or self.__iNumberTestCaseFailure != 0: EDVerbose.unitTest(" Total number of test cases executed with SUCCESS : %d" % self.__iNumberTestCaseSuccess) EDVerbose.unitTest(" Total number of test cases executed with FAILURE : %d" % self.__iNumberTestCaseFailure) EDVerbose.unitTest() if self.__dictTestCaseFailureMessages != {}: EDVerbose.unitTest("") EDVerbose.unitTest("OBS! The following test methods ended with failure:") EDVerbose.unitTest("") for strTestCaseName in self.__dictTestCaseFailureMessages.keys(): EDVerbose.unitTest(" %s :" % strTestCaseName) for strMethodName in self.__dictTestCaseFailureMessages[strTestCaseName].keys(): EDVerbose.unitTest(" %s :" % strMethodName.split(".")[1]) EDVerbose.unitTest(" %s" % self.__dictTestCaseFailureMessages[strTestCaseName][strMethodName]) EDVerbose.unitTest("") EDVerbose.unitTest("") if self.__iNumberTestMethodSuccess != 0 or self.__iNumberTestMethodFailure != 0: EDVerbose.unitTest("Total number of test methods executed with SUCCESS : %d" % self.__iNumberTestMethodSuccess) EDVerbose.unitTest("Total number of test methods executed with FAILURE : %d" % self.__iNumberTestMethodFailure) EDVerbose.unitTest() EDVerbose.unitTest(" Runtime : %.3f [s]" % self.getRunTime()) EDVerbose.unitTest("###################################################################")
def preProcess(self): """ Creates the application working directory (log dir) Initializes the configuration retrieves the plugin xml data to be passed to the plugin """ EDVerbose.DEBUG("EDApplication.preProcess") self.processCommandline() if (not self.__bIsFailure): # Check that the plugin can be located strPluginLocation = EDFactoryPluginStatic.getFactoryPlugin().getModuleLocation(self.__strPluginName) if (strPluginLocation is None): EDVerbose.error("Plugin %s cannot be loaded!" % self.__strPluginName) self.__bIsFailure = True # Check that the input file can be read if (self.getDataInputFilePath() is not None) and (not os.path.exists(self.__strDataInputFilePath)): EDVerbose.error("Input XML file not found : %s" % self.__strDataInputFilePath) self.__bIsFailure = True # Check that the output file can be created if (self.__strDataOutputFilePath is not None): strOutputDirectory = os.path.dirname(self.__strDataOutputFilePath) if (strOutputDirectory is None or strOutputDirectory == ""): strOutputDirectory = os.getcwd() self.__strDataOutputFilePath = os.path.join(strOutputDirectory, self.__strDataOutputFilePath) if (not os.access(strOutputDirectory, os.W_OK)): EDVerbose.error("Output directory not writable: %s" % strOutputDirectory) self.__bIsFailure = True elif (os.path.exists(self.__strDataOutputFilePath)): if (not os.access(self.__strDataOutputFilePath, os.W_OK)): EDVerbose.error("Output file not writable: %s" % self.__strDataOutputFilePath) self.__bIsFailure = True if (not self.__bIsFailure): EDVerbose.DEBUG("EDApplication.PLUGIN_PARAM_LABEL: " + EDApplication.PLUGIN_PARAM_LABEL) # Load the configuration file if(self.__strConfigurationFileName is None): self.__strConfigurationHome = EDApplication.getConfigurationHome(self.__strPluginName) self.__strConfigurationFileName = os.path.abspath(os.path.join(self.__strConfigurationHome, "XSConfiguration_%s.xml" % EDUtilsPath.getEdnaSite())) if (os.path.exists(self.__strConfigurationFileName)): EDVerbose.screen("Loading Configuration file: %s" % self.__strConfigurationFileName) edConfiguration = EDConfiguration(self.__strConfigurationFileName) self.setConfiguration(edConfiguration) self.loadConfiguration() EDVerbose.DEBUG("EDApplication.preProcess: Checking... Number of plugins...: %d" % EDApplication.__edConfiguration.getPluginListSize()) pyDictionary = {} pyDictionary[ "${EDNA_HOME}" ] = EDUtilsPath.getEdnaHome() if self.getDataInputFilePath() is not None: self.__strXMLData = EDUtilsFile.readFileAndParseVariables(self.getDataInputFilePath(), pyDictionary) else: EDVerbose.warning("Cannot find configuration file: %s" % self.__strConfigurationFileName) # Create the application working directory if(self.__strWorkingDir is None): self.__strWorkingDir = self.__strApplicationInstanceName self.createApplicationWorkingDirectory()
def functionXMLout(_strXMLin, _strXMLout): """ This is an example of XMLout function ... it prints only the name of the file created @param _srXMLin: The XML string used to launch the job @type _strXMLin: python string with the input XML @param _strXMLout: The XML string retrieved job @type _strXMLout: python string with the output XML @return: None """ xsDataInputGridScreening = XSDataInputGridScreening.parseString(_strXMLin) strImagePath = xsDataInputGridScreening.getImageFile().getPath().getValue() EDVerbose.screen("Successful processing of image %s" % strImagePath) strResultText = "%40s " % os.path.basename(strImagePath) xsDataResultGridScreening = XSDataResultGridScreening.parseString(_strXMLout) fileNameParameters = xsDataResultGridScreening.getFileNameParameters() if fileNameParameters is None: strResultText += "%6s%10s%10s%6s" % ("NA", "NA", "NA","NA") else: strResultText += "%6s%10s%10s%6s" % ( fileNameParameters.getScanId1().getValue(), fileNameParameters.getMotorPosition1().getValue(), fileNameParameters.getMotorPosition2().getValue(), fileNameParameters.getScanId2().getValue(), ) imageQualityIndicators = xsDataResultGridScreening.getImageQualityIndicators() if imageQualityIndicators is None: strResultText += "%6s%6s%6s%6s%10s" % ("NA", "NA", "NA","NA", "NA") else: strMethod1Res = "%6s" % "NA" if imageQualityIndicators.getMethod1Res(): strMethod1Res = "%6.1f" % imageQualityIndicators.getMethod1Res().getValue() strMethod2Res = "%6s" % "NA" if imageQualityIndicators.getMethod2Res(): strMethod2Res = "%6.1f" % imageQualityIndicators.getMethod2Res().getValue() strSpotTotal = "%6s" % "NA" if imageQualityIndicators.getSpotTotal(): strSpotTotal = "%6d" % imageQualityIndicators.getSpotTotal().getValue() strGoodBraggCandidates = "%6s" % "NA" if imageQualityIndicators.getGoodBraggCandidates(): strGoodBraggCandidates = "%6d" % imageQualityIndicators.getGoodBraggCandidates().getValue() strTotalIntegratedSignal = "%10s" % "NA" if imageQualityIndicators.getTotalIntegratedSignal(): strTotalIntegratedSignal = "%10.0f" % imageQualityIndicators.getTotalIntegratedSignal().getValue() strResultText += strMethod1Res + strMethod2Res + strSpotTotal + strGoodBraggCandidates + strTotalIntegratedSignal if xsDataResultGridScreening.getMosaicity() is None: strResultText += "%6s" % "NA" else: strResultText += "%6.1f" % xsDataResultGridScreening.getMosaicity().getValue() if xsDataResultGridScreening.getRankingResolution() is None: strResultText += "%6s" % "NA" else: strResultText += "%6.1f" % xsDataResultGridScreening.getRankingResolution().getValue() strResultText += " " + xsDataResultGridScreening.getComment().getValue() writeToResultFile(strResultText)
def doFailureActionSubWedgeMerge(self, _edPlugin=None): EDVerbose.DEBUG( "EDPluginControlSubWedgeAssemblev1_1.doFailureActionSubWedgeMerge") EDVerbose.screen("Execution of " + self.__strPluginSubWedgeMergeName + " failed.") EDVerbose.screen( "Please inspect the log file for further information.") self.retrieveFailureMessages( _edPlugin, "EDPluginControlSubWedgeAssemblev1_1.doFailureActionSubWedgeMerge") self.setFailure()
def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ EDUtilsParallel.semaphoreNbThreadsRelease() self.__bIsFirstExecute = False if self.__functXMLout is None: EDVerbose.screen("Plugin %s execution ended with success" % self.__strPluginName) else: self.__semaphoreOut.acquire() self.__functXMLout(_edObject.dataInput.marshal(), _edObject.getDataOutput().marshal()) self.__semaphoreOut.release()
def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ EDUtilsParallel.semaphoreNbThreadsRelease() self.__bIsFirstExecute = False if self.__functXMLerr is None: EDVerbose.screen("Plugin %s execution ended with failure" % self.__strPluginName) else: self.__semaphoreErr.acquire() self.__functXMLerr(_edObject.dataInput.marshal()) self.__semaphoreErr.release()
def makeXML(self, filename): """Here we create the XML string to be passed to the EDNA plugin from the input filename This can / should be modified by the final user @param filename: full path of the input file @type filename: python string representing the path @rtype: XML string @return: python string """ bProcessFile = False basename = os.path.basename(filename) for oneExt in self.listExtensions: if basename.endswith(oneExt): bProcessFile = True break if bProcessFile: for onePref in self.listExcludedPrefix: if basename.startswith(onePref): bProcessFile = False if not bProcessFile: EDVerbose.screen("Not processing file: %s" % filename) return xsdimage = XSDataImage() xsdimage.setPath(XSDataString(filename)) xsd = XSDataInputDiffractionCT() xsd.setImage(xsdimage) xsd.setDestinationDirectory(self.destinationDirectory) xsd.setSinogramFileNamePrefix(self.sinogramFileNamePrefix) xsd.setPowderDiffractionSubdirectory( self.powderDiffractionSubdirectory) xsd.setPowderDiffractionFormat(self.powderDiffractionFormat) if self.indexOffset is not None and self.fastMotorSteps is not None: idx = 0 basename = list(os.path.splitext(os.path.basename(filename))[0]) basename.reverse() number = "" for i in basename: if i.isdigit(): number = i + number else: break idx = int(number) - self.indexOffset if idx < 0: return self.forceInstrument.set_tomo_spec_displ_x( XSDataLength(idx % (self.fastMotorSteps + 1))) self.forceInstrument.set_tomo_spec_displ_rotation( XSDataLength(idx // (self.fastMotorSteps + 1))) xsd.setOverrideScanParam(self.forceInstrument) xsd.setOverrideImageParam(self.forceImage) return xsd.marshal()
def processCommandLineItems(self, _strCommandLineItem, _bFoundImageToken): bFoundImageToken = _bFoundImageToken if (_strCommandLineItem == EDApplicationMXv1Characterisation.TEMPLATE_PARAM_LABEL): self.__bTemplateMode = True self.__strGeneratedTemplateFile = self.getCommandLineArgument( EDApplicationMXv1Characterisation.TEMPLATE_PARAM_LABEL) if (self.__strGeneratedTemplateFile == None): strErrorMessage = EDMessage.ERROR_MANDATORY_PARAM_MISSING_02 % ("EDApplicationMXv1Characterisation.preProcess", "No argument for command line %s key word found!" % \ EDApplicationMXv1Characterisation.TEMPLATE_PARAM_LABEL) EDVerbose.error(strErrorMessage) return False if (not os.path.isabs(self.__strGeneratedTemplateFile)): self.__strGeneratedTemplateFile = os.path.abspath( os.path.join(self.getCurrentWorkingDirectory(), self.__strGeneratedTemplateFile)) if (_strCommandLineItem == EDApplicationMXv1Characterisation.ANOMALOUS_DATA_LABEL): self.__bAnomalousData = True EDVerbose.screen( "Anomalous data: : True") if (bFoundImageToken == False): if (_strCommandLineItem == EDApplicationMXv1Characterisation.IMAGE_PARAM_LABEL): bFoundImageToken = True else: if (_strCommandLineItem[0:2] == "--"): bFoundImageToken = False else: if (self.__listImagePaths is None): self.__listImagePaths = [] if (not os.path.isabs(_strCommandLineItem)): strAbsolutePath = os.path.abspath( os.path.join(self.getCurrentWorkingDirectory(), _strCommandLineItem)) else: strAbsolutePath = _strCommandLineItem if (not os.path.exists(strAbsolutePath)): strErrorMessage = EDMessage.ERROR_CANNOT_READ_FILE_02 % ( "EDApplicationMXv1Characterisation.readAndProcessCommandLine", strAbsolutePath) EDVerbose.error(strErrorMessage) return False else: EDVerbose.DEBUG( "EDApplicationMXv1Characterisation.readAndProcessCommandLine: Found image path: " + strAbsolutePath) self.__listImagePaths.append(strAbsolutePath) return bFoundImageToken
def showMessage(_iLevel, _strMessage, _strFilename=None): """ Class Method: Similar to logging module of python but for updating BioSaxsCube @param _iLevel: print level seems to be 4 for Errors 3 for Warnings 2 for Info 1 0 @type _iLevel: int @param _strMessage: comment to be printed @type _strMessage: string @param _strFilename: the file related to the message (nothing to do with a logfile) @type _strFilename: string or None """ if _iLevel == 4: EDVerbose.ERROR(_strMessage) elif _iLevel == 3: EDVerbose.WARNING(_strMessage) else: EDVerbose.screen(_strMessage) # else: # EDVerbose.DEBUG(_strMessage) if EDUtilsBioSaxs.specStatus is not None: currentStatus = EDUtilsBioSaxs.specStatus.value["reprocess"][ "status"] # must do this, since SpecClient is apparently returning a non-expected data structure i = currentStatus.rfind(",") # TB: This ,1 or ,0 suffix nonsense seems to be a hack to force Spec to signal a variable change to bsxcube if i == -1 or currentStatus[i + 1:] == "1": if _strFilename is None: newStatus = "%s,%s,0" % (_iLevel, _strMessage) else: newStatus = "%s,%s,%s,0" % (_iLevel, _strMessage, _strFilename) else: if _strFilename is None: newStatus = "%s,%s,1" % (_iLevel, _strMessage) else: newStatus = "%s,%s,%s,1" % (_iLevel, _strMessage, _strFilename) EDUtilsBioSaxs.specStatus.setValue(newStatus) if (EDUtilsBioSaxs.specAbort is not None) and ( EDUtilsBioSaxs.specAbort.value["reprocess"]["abort"]) == "1": # must do this, since SpecClient is apparently returning a non-expected data structure EDVerbose.ERROR("Aborting data reprocess!")
def doSuccessActionPlugin(self, _edPlugin=None): """ retrieve the potential warning messages """ EDApplication.doSuccessActionPlugin(self, _edPlugin) EDVerbose.DEBUG("EDApplicationXAFSBatchProcessing.doSuccessActionPlugin") if (_edPlugin.getListOfErrorMessages() != []): self.doFailureActionPlugin(_edPlugin) elif (_edPlugin.getListOfWarningMessages() != []): EDVerbose.screen("XAFS data processing successful with warning messages, please check the log file.") else: EDVerbose.screen("XAFS data processing successful!")
def preProcess(self): """ Creates the application working directory (log dir) Initializes the configuration retrieves the plugin xml data to be passed to the plugin """ EDVerbose.DEBUG("EDApplication.preProcess") self.processCommandline() if (not self._bIsFailure): # Check that the plugin can be located strPluginLocation = EDFactoryPluginStatic.getFactoryPlugin().getModuleLocation(self._strPluginName) if (strPluginLocation is None): EDVerbose.error("Plugin %s cannot be loaded!" % self._strPluginName) self._bIsFailure = True # Check that the input file can be read if (self.getDataInputFilePath() is not None) and (not os.path.exists(self._strDataInputFilePath)): EDVerbose.error("Input XML file not found : %s" % self._strDataInputFilePath) self._bIsFailure = True # Check that the output file can be created if (self._strDataOutputFilePath is not None): strOutputDirectory = os.path.dirname(self._strDataOutputFilePath) if (strOutputDirectory is None or strOutputDirectory == ""): strOutputDirectory = os.getcwd() self._strDataOutputFilePath = os.path.join(strOutputDirectory, self._strDataOutputFilePath) if (not os.access(strOutputDirectory, os.W_OK)): EDVerbose.error("Output directory not writable: %s" % strOutputDirectory) self._bIsFailure = True elif (os.path.exists(self._strDataOutputFilePath)): if (not os.access(self._strDataOutputFilePath, os.W_OK)): EDVerbose.error("Output file not writable: %s" % self._strDataOutputFilePath) self._bIsFailure = True if (not self._bIsFailure): EDVerbose.DEBUG("EDApplication.PLUGIN_PARAM_LABEL: " + EDApplication.PLUGIN_PARAM_LABEL) if self._strConfigurationFileName is not None: # Load the configuration file if (os.path.exists(self._strConfigurationFileName)): EDVerbose.screen("Loading Configuration file: %s" % self._strConfigurationFileName) EDConfigurationStatic.addConfigurationFile(self._strConfigurationFileName, _bReplace=True) else: EDVerbose.warning("Cannot find configuration file: %s" % self._strConfigurationFileName) pyDictionary = {} pyDictionary[ "${EDNA_HOME}" ] = EDUtilsPath.getEdnaHome() if self.getDataInputFilePath() is not None: self._strXMLData = EDUtilsFile.readFileAndParseVariables(self.getDataInputFilePath(), pyDictionary) # Create the application working directory if(self._strWorkingDir is None): self._strWorkingDir = self._strApplicationInstanceName self.createApplicationWorkingDirectory()
def processKernel(self): """ Executes the test case. """ EDVerbose.DEBUG("EDTestCase.processKernel") EDVerbose.screen() EDVerbose.unitTest("===================================================================") if self.getTestSuiteName() is not None: EDVerbose.unitTest("TEST SUITE : %s" % self.getTestSuiteName()) EDVerbose.unitTest("TEST CASE : %s" % self.getClassName()) EDVerbose.unitTest(" ") if self.__strReasonForNotBeingExecuted == "": self.setTimeInit() iNumberMethods = self.getNumberOfTests() EDVerbose.unitTest("Total number of tests : %d" % iNumberMethods) EDVerbose.unitTest() iTestCaseNumber = 0 if self.getListTest() == []: self.__strReasonForNotBeingExecuted = "No test methods!" else: for pyTestMethod in self.getListTest(): iTestCaseNumber = iTestCaseNumber + 1 strMethodName = EDUtilsTest.patchMethodName(pyTestMethod) EDVerbose.unitTest("-------------------------------------------------------------------") EDVerbose.unitTest("Test case method : %s" % strMethodName) EDVerbose.unitTest() try: pyTestMethod() self.__iNumberTestMethodSuccess += 1 EDVerbose.unitTest("%s executed with SUCCESS" % strMethodName) EDVerbose.unitTest() except AssertionError, pyException: self.__dictMethodFailureMessages[strMethodName] = str(pyException) self.__iNumberTestMethodFailure += 1 EDVerbose.unitTest("Assertion Error Raised!") EDVerbose.unitTest("%s executed with FAILURE" % strMethodName) EDVerbose.unitTest() except Exception, pyException: self.__dictMethodFailureMessages[strMethodName] = str(pyException) self.__iNumberTestMethodFailure += 1 EDVerbose.unitTest("Unexpected Error!") EDVerbose.unitTest("%s executed with FAILURE" % strMethodName) EDVerbose.unitTest() EDVerbose.writeErrorTrace() EDVerbose.unitTest() self.__bIsExecuted = True self.setTimeEnd()
def makeXML(self, filename): """Here we create the XML string to be passed to the EDNA plugin from the input filename This can / should be modified by the final user @param filename: full path of the input file @type filename: python string representing the path @rtype: XML string @return: python string """ bProcessFile = False basename = os.path.basename(filename) for oneExt in self.listExtensions: if basename.endswith(oneExt): bProcessFile = True break if bProcessFile: for onePref in self.listExcludedPrefix: if basename.startswith(onePref): bProcessFile = False if not bProcessFile: EDVerbose.screen("Not processing file: %s" % filename) return xsdimage = XSDataImage() xsdimage.setPath(XSDataString(filename)) xsd = XSDataInputDiffractionCT() xsd.setImage(xsdimage) xsd.setDestinationDirectory(self.destinationDirectory) xsd.setSinogramFileNamePrefix(self.sinogramFileNamePrefix) xsd.setPowderDiffractionSubdirectory(self.powderDiffractionSubdirectory) xsd.setPowderDiffractionFormat(self.powderDiffractionFormat) if self.indexOffset is not None and self.fastMotorSteps is not None: idx = 0 basename = list(os.path.splitext(os.path.basename(filename))[0]) basename.reverse() number = "" for i in basename: if i.isdigit(): number = i + number else: break idx = int(number) - self.indexOffset if idx < 0:return self.forceInstrument.set_tomo_spec_displ_x(XSDataLength(idx % (self.fastMotorSteps + 1))) self.forceInstrument.set_tomo_spec_displ_rotation(XSDataLength(idx // (self.fastMotorSteps + 1))) xsd.setOverrideScanParam(self.forceInstrument) xsd.setOverrideImageParam(self.forceImage) return xsd.marshal()
def doSuccessEvaluationIndexingLABELIT(self, _edPlugin=None): EDVerbose.DEBUG("EDPluginControlCharacterisationv1_2.doSuccessEvaluationIndexingLABELIT") self.retrieveSuccessMessages(_edPlugin, "EDPluginControlCharacterisationv1_2.doSuccessEvaluationIndexingLABELIT") # Retrieve status messages (if any) if self.__edPluginExecEvaluationIndexingLABELIT.hasDataOutput("statusMessageIndexing"): self.addStatusMessage("Labelit: " + self.__edPluginExecEvaluationIndexingLABELIT.getDataOutput("statusMessageIndexing")[0].getValue()) # Check if indexing was successful bIndexingSuccess = self.__edPluginExecEvaluationIndexingLABELIT.getDataOutput("indexingSuccess")[0].getValue() if bIndexingSuccess: xsDataIndexingResult = self.__edPluginExecEvaluationIndexingLABELIT.getDataOutput("indexingResult")[0] self.__xsDataResultCharacterisation.setIndexingResult(xsDataIndexingResult) # Then start the integration of the reference images self.indexingToIntegration() else: EDVerbose.screen("Indexing with LABELIT failed!") self.setFailure()
def doSuccessActionPlugin(self, _edPlugin=None): """ retrieve the potential warning messages and set the output """ EDApplication.doSuccessActionPlugin(self, _edPlugin) EDVerbose.DEBUG("EDApplicationDimplev0.doSuccessActionPlugin") xsDataOutput = _edPlugin.getDataOutput() if (_edPlugin.getListOfErrorMessages() != []): self.doFailureActionPlugin(_edPlugin) elif (_edPlugin.getListOfWarningMessages() != []): EDVerbose.screen("Dimplev0 successful with warning messages, please check the log file.") else: EDVerbose.screen("Dimplev0 successful!")
def process(self): for fn in self.dataFiles: EDVerbose.screen("Processing file %s" % fn) edj = EDJob(self.EDNAPluginName) edj.dataInput = self.fileName2xml(fn) edj.connectSUCCESS(self.XMLsuccess) edj.connectFAILURE(self.XMLerr) self.queue.put(edj) if self.process_sem._Semaphore__value > 0 : t = threading.Thread(target=self.startProcessing) t.start() EDVerbose.screen("Back in main") while self.cpu_sem._Semaphore__value < self.nbcpu: time.sleep(0.1) EDJob.synchronizeAll() EDJob.stats()
def showMessage(_iLevel, _strMessage, _strFilename=None): """ Class Method: Similar to logging module of python but for updating BioSaxsCube @param _iLevel: print level seems to be 4 for Errors 3 for Warnings 2 for Info 1 0 @type _iLevel: int @param _strMessage: comment to be printed @type _strMessage: string @param _strFilename: the file related to the message (nothing to do with a logfile) @type _strFilename: string or None """ if _iLevel == 4: EDVerbose.ERROR(_strMessage) elif _iLevel == 3: EDVerbose.WARNING(_strMessage) else: EDVerbose.screen(_strMessage) # else: # EDVerbose.DEBUG(_strMessage) if EDUtilsBioSaxs.specStatus is not None: currentStatus = EDUtilsBioSaxs.specStatus.value["reprocess"]["status"] # must do this, since SpecClient is apparently returning a non-expected data structure i = currentStatus.rfind(",") # TB: This ,1 or ,0 suffix nonsense seems to be a hack to force Spec to signal a variable change to bsxcube if i == -1 or currentStatus[i + 1:] == "1": if _strFilename is None: newStatus = "%s,%s,0" % (_iLevel, _strMessage) else: newStatus = "%s,%s,%s,0" % (_iLevel, _strMessage, _strFilename) else: if _strFilename is None: newStatus = "%s,%s,1" % (_iLevel, _strMessage) else: newStatus = "%s,%s,%s,1" % (_iLevel, _strMessage, _strFilename) EDUtilsBioSaxs.specStatus.setValue(newStatus) if (EDUtilsBioSaxs.specAbort is not None) and (EDUtilsBioSaxs.specAbort.value["reprocess"]["abort"]) == "1": # must do this, since SpecClient is apparently returning a non-expected data structure EDVerbose.ERROR("Aborting data reprocess!")
def testGenerateSPDCommand(self): edPluginSPD = self.createPlugin() strXMLInput = EDUtilsFile.readFileAndParseVariables(self.strReferenceInputFileName, EDUtilsPath.getDictOfPaths()) xsDataInputSPDCake = XSDataInputSPDCake.parseString(strXMLInput) edPluginSPD.setDataInput(xsDataInputSPDCake) edPluginSPD.configure() edPluginSPD.getInputParameter() #edPluginSPD.preProcess() edPluginSPD.generateSPDCommand() EDVerbose.screen("SPD configuration:\n%s" % edPluginSPD.getSPDConfig()) if EDVerbose.isVerboseDebug(): expected = """off_1=0 off_2=0 verbose=2 src_ext=.edf cor_ext=.cor wvl=1.000000e-10 cen_1=1.050000e+03 cen_2=1.000000e+03 dis=1.000000e-01 pix_1=4.722440e-05 pix_2=4.683150e-05 do_distortion=0 do_dark=0""" #"do_distortion=2 off_1=0 off_2=0 verbose=2 src_ext=.edf cor_ext=.cor wvl=1.000000e-10 cen_1=1.050000e+03 cen_2=1.000000e+03 dis=1.000000e-01 pix_1=4.722440e-05 pix_2=4.683150e-05 do_dark=0" else: expected = """off_1=0 off_2=0 verbose=0 src_ext=.edf cor_ext=.cor wvl=1.000000e-10 cen_1=1.050000e+03 cen_2=1.000000e+03 dis=1.000000e-01 pix_1=4.722440e-05 pix_2=4.683150e-05 do_distortion=0 do_dark=0""" #do_distortion=2 off_1=0 off_2=0 verbose=0 src_ext=.edf cor_ext=.cor wvl=1.000000e-10 cen_1=1.050000e+03 cen_2=1.000000e+03 dis=1.000000e-01 pix_1=4.722440e-05 pix_2=4.683150e-05 do_dark=0" EDAssert.equal(edPluginSPD.getSPDConfig(), expected)
def failurePluginExecution(self, _edObject=None): """ Method called when the execution of the plugin failed """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_FAILURE EDVerbose.screen("Plugin %s execution ended with failure" % self.__jobId) self.synchronizeOff() try: self.__edSlotFAILURE.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Failure call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Common call-back (after failure) for %s" % self.__jobId) EDVerbose.writeErrorTrace()
def successPluginExecution(self, _edObject=None): """ Method called when the execution of the plugin succeeds """ self.synchronizeOn() self.__status = EDJob.PLUGIN_STATE_SUCCESS EDVerbose.screen("Plugin %s execution ended with success" % self.__jobId) self.synchronizeOff() try: self.__edSlotSUCCESS.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Success call-back for %s" % self.__jobId) EDVerbose.writeErrorTrace() try: self.__edSlotCallBack.call(self.__jobId) except Exception: EDVerbose.ERROR("Error in execution of Common call-back (after success) for %s" % self.__jobId) EDVerbose.writeErrorTrace()