def createDataInputCharacterisationFromDataSets(self, _pyListXSDataCCP4iDataSet): """ This method takes as input a list of ccp4i data sets. Each data set can contain several paths to image files. It runs the EDPluginControlSubWedgeAssemble plugin to read the experimental information from the image headers and then creates a single XSDataInputCharacterisation object. """ EDVerbose.DEBUG( "EDPluginControlCCP4iv1_1.createDataInputCharacterisationFromDataSets" ) xsDataInputCharacterisation = None # We might have to run the plugin several times edActionCluster = EDActionCluster() pyListPluginControlSubWedgeAssemble = [] # Prepare the action cluster iIndex = 1 for xsDataCCP4iDataSet in _pyListXSDataCCP4iDataSet: edPluginControlSubWedgeAssemble = self.loadPlugin( self.__strPluginControlSubWedgeAssembleName, "SubWedgeAssemble-%02d" % iIndex) edPluginControlSubWedgeAssemble.connectSUCCESS( self.doSuccessActionSubWedgeAssemble) edPluginControlSubWedgeAssemble.connectFAILURE( self.doFailureActionSubWedgeAssemble) # Prepare the input for the sub wedge assemble plugin xsDataInputSubWedgeAssemble = XSDataInputSubWedgeAssemble() for xsDataImageFile in xsDataCCP4iDataSet.getImageFile(): xsDataInputSubWedgeAssemble.addFile(xsDataImageFile) edPluginControlSubWedgeAssemble.setDataInput( xsDataInputSubWedgeAssemble) pyListPluginControlSubWedgeAssemble.append( edPluginControlSubWedgeAssemble) self.addPluginToActionCluster(edPluginControlSubWedgeAssemble) iIndex += 1 # Run the action cluster synchronously self.executeActionCluster() self.synchronizeActionCluster() # Recuperate the output for edPluginControlSubWedgeAssemble in pyListPluginControlSubWedgeAssemble: if (edPluginControlSubWedgeAssemble.isFailure()): self.setFailure() else: xsDataResultSubWedgeAssemble = edPluginControlSubWedgeAssemble.getDataOutput( ) for xsDataSubWedge in xsDataResultSubWedgeAssemble.getSubWedge( ): # Instantiate the xsDataInputCharacterisation object if it's not already done. xsDataCollection = None if (xsDataInputCharacterisation is None): xsDataInputCharacterisation = XSDataInputCharacterisation( ) else: xsDataCollection = xsDataInputCharacterisation.getDataCollection( ) if (xsDataCollection is None): xsDataCollection = XSDataCollection() xsDataCollection.addSubWedge(xsDataSubWedge) xsDataInputCharacterisation.setDataCollection( xsDataCollection) return xsDataInputCharacterisation
def doXdsIndexingIntegration(self, _xsDataCollection): # Load the plugin self._edPluginIndexing = self.loadPlugin(self._strIndexingPluginName, "Indexing") # XDS Indexing xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(_xsDataCollection) self._edPluginIndexing.dataInput = EDHandlerXSDataXDSv1_0.generateXSDataInputXDSIndexing(xsDataIndexingInput) self._edPluginIndexing.executeSynchronous() xsDataResultXDSIndexing = self._edPluginIndexing.dataOutput if xsDataResultXDSIndexing.spaceGroupNumber is not None: spaceGroupNumber = xsDataResultXDSIndexing.spaceGroupNumber.value unitCell = xsDataResultXDSIndexing.unitCell filePaths = xsDataResultXDSIndexing.filePaths index = 1 for subWedge in _xsDataCollection.subWedge: xsDataCollection = XSDataCollection() xsDataCollection.addSubWedge(subWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(xsDataCollection) xsDataInputXDSIntegration = EDHandlerXSDataXDSv1_0.generateXSDataInputXDSIntegration(xsDataIndexingInput, spaceGroupNumber, unitCell, filePaths) edPluginIntegration = self.loadPlugin(self._strIntegrationPluginName, "Integration_{0}".format(index)) edPluginIntegration.dataInput = xsDataInputXDSIntegration edPluginIntegration.executeSynchronous() # self._edPluginIntegration.dataInput = xsDataInputXDSIntegration index += 1
def createDataCollectionFromDataFiles(self, _pyListXSDataFile): """ This method takes as input a list of paths to XML data files. It parses these files and create a single XSDataCollection object. """ EDVerbose.DEBUG( "EDPluginControlCCP4iv10.createDataCollectionFromDataFiles") xsDataCollection = None xsDataCollectionCurrent = None for xsDataInputFile in _pyListXSDataFile: try: strInputFileContent = EDUtilsFile.readFile( xsDataInputFile.getPath().getValue()) xsDataCollectionCurrent = XSDataCollection.parseString( strInputFileContent) except Exception, detail: errorMessage = EDMessage.ERROR_EXECUTION_03 % ( 'EDPluginControlCCP4iv10.preProcess', 'EDPluginControlCCP4iv10', detail) EDVerbose.error(errorMessage) self.addErrorMessage(errorMessage) self.setFailure() if (xsDataCollectionCurrent is None): errorMessage = EDMessage.ERROR_EXECUTION_03 % ( 'EDPluginControlCCP4iv10.preProcess', 'EDPluginControlCCP4iv10', "None data collection") EDVerbose.error(errorMessage) self.addErrorMessage(errorMessage) self.setFailure() else: # Instantiate the xsDataCollection object if it's not already done. if (xsDataCollection is None): xsDataCollection = XSDataCollection() for xsDataSubWedge in xsDataCollectionCurrent.getSubWedge(): xsDataCollection.addSubWedge(xsDataSubWedge)
def doXdsIndexingIntegration(self, _xsDataCollection): # Load the plugin self._edPluginIndexing = self.loadPlugin(self._strIndexingPluginName, "Indexing") # XDS Indexing xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(_xsDataCollection) self._edPluginIndexing.dataInput = EDHandlerXSDataXDSv1_0.generateXSDataInputXDSIndexing( xsDataIndexingInput) self._edPluginIndexing.executeSynchronous() xsDataResultXDSIndexing = self._edPluginIndexing.dataOutput if xsDataResultXDSIndexing.spaceGroupNumber is not None: spaceGroupNumber = xsDataResultXDSIndexing.spaceGroupNumber.value unitCell = xsDataResultXDSIndexing.unitCell filePaths = xsDataResultXDSIndexing.filePaths index = 1 for subWedge in _xsDataCollection.subWedge: xsDataCollection = XSDataCollection() xsDataCollection.addSubWedge(subWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(xsDataCollection) xsDataInputXDSIntegration = EDHandlerXSDataXDSv1_0.generateXSDataInputXDSIntegration( xsDataIndexingInput, spaceGroupNumber, unitCell, filePaths) edPluginIntegration = self.loadPlugin( self._strIntegrationPluginName, "Integration_{0}".format(index)) edPluginIntegration.dataInput = xsDataInputXDSIntegration edPluginIntegration.executeSynchronous() # self._edPluginIntegration.dataInput = xsDataInputXDSIntegration index += 1
def createDataCollectionFromDataFiles(self, _pyListXSDataFile): """ This method takes as input a list of paths to XML data files. It parses these files and create a single XSDataCollection object. """ EDVerbose.DEBUG("EDPluginControlCCP4iv10.createDataCollectionFromDataFiles") xsDataCollection = None xsDataCollectionCurrent = None for xsDataInputFile in _pyListXSDataFile: try: strInputFileContent = EDUtilsFile.readFile(xsDataInputFile.getPath().getValue()) xsDataCollectionCurrent = XSDataCollection.parseString(strInputFileContent) except Exception, detail: errorMessage = EDMessage.ERROR_EXECUTION_03 % ('EDPluginControlCCP4iv10.preProcess', 'EDPluginControlCCP4iv10', detail) EDVerbose.error(errorMessage) self.addErrorMessage(errorMessage) self.setFailure() if (xsDataCollectionCurrent is None): errorMessage = EDMessage.ERROR_EXECUTION_03 % ('EDPluginControlCCP4iv10.preProcess', 'EDPluginControlCCP4iv10', "None data collection") EDVerbose.error(errorMessage) self.addErrorMessage(errorMessage) self.setFailure() else: # Instantiate the xsDataCollection object if it's not already done. if (xsDataCollection is None): xsDataCollection = XSDataCollection() for xsDataSubWedge in xsDataCollectionCurrent.getSubWedge(): xsDataCollection.addSubWedge(xsDataSubWedge)
def preProcess(self, _edObject=None): """ Gets the Configuration Parameters, if found, overrides default parameters """ EDPluginControl.preProcess(self, _edObject) self.DEBUG("EDPluginControlGeneratePredictionv10.preProcess...") xsDataGeneratePredictionInput = self.getDataInput() xsDataSelectedIndexingSolution = xsDataGeneratePredictionInput.getSelectedIndexingSolution() xsDataExperimentalConditionRefined = xsDataSelectedIndexingSolution.getExperimentalConditionRefined() xsDataCollection = xsDataGeneratePredictionInput.getDataCollection() xsDataSubWedgeList = xsDataCollection.getSubWedge() # List containing instances of all the generate prediction plugins self.__listPluginGeneratePrediction = [] # Loop through all subwedges iIndex = 0 for xsDataSubWedge in xsDataSubWedgeList: xsDataImageList = xsDataSubWedge.getImage() # First find the lowest image number iLowestImageNumber = None for xsDataImage in xsDataImageList: iImageNumber = xsDataImage.getNumber().getValue() if (iLowestImageNumber is None): iLowestImageNumber = iImageNumber elif (iImageNumber < iLowestImageNumber): iLowestImageNumber = iImageNumber # Then loop through all images in a sub wedge for xsDataImage in xsDataImageList: iIndex += 1 edPluginGeneratePrediction = self.loadPlugin(self.__strPluginGeneratePredictionName, "%s-%02d" % (self.__strPluginGeneratePredictionName, iIndex)) xsDataGeneratePredictionInput = XSDataGeneratePredictionInput() xsDataGeneratePredictionInput.setSelectedIndexingSolution(XSDataIndexingSolutionSelected.parseString(xsDataSelectedIndexingSolution.marshal())) xsDataCollectionNew = XSDataCollection() xsDataSubWedgeNew = XSDataSubWedge() xsDataSubWedgeNew.addImage(XSDataImage.parseString(xsDataImage.marshal())) xsDataSubWedgeNew.setExperimentalCondition(XSDataExperimentalCondition.parseString(xsDataSubWedge.getExperimentalCondition().marshal())) # We must modify the rotationOscillationStart for the new subwedge xsDataGoniostatNew = xsDataSubWedgeNew.getExperimentalCondition().getGoniostat() fGoniostatRotationAxisStart = xsDataGoniostatNew.getRotationAxisStart().getValue() fGonioStatOscillationRange = xsDataGoniostatNew.getOscillationWidth().getValue() iImageNumber = xsDataImage.getNumber().getValue() fGoniostatRotationAxisStartNew = fGoniostatRotationAxisStart + (iImageNumber - iLowestImageNumber) * fGonioStatOscillationRange xsDataGoniostatNew.setRotationAxisStart(XSDataAngle(fGoniostatRotationAxisStartNew)) # xsDataCollectionNew.addSubWedge(xsDataSubWedgeNew) xsDataGeneratePredictionInput.setDataCollection(xsDataCollectionNew) from EDHandlerXSDataMOSFLMv10 import EDHandlerXSDataMOSFLMv10 xsDataMOSFLMInputGeneratePrediction = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputGeneratePrediction(xsDataGeneratePredictionInput) edPluginGeneratePrediction.setDataInput(xsDataMOSFLMInputGeneratePrediction) self.__listPluginGeneratePrediction.append(edPluginGeneratePrediction)
def preProcess(self, _edObject=None): """ Gets the Configuration Parameters, if found, overrides default parameters """ EDPluginControl.preProcess(self, _edObject) self.DEBUG("EDPluginControlIntegrationv10.preProcess...") xsDataIntegrationInput = self.getDataInput() xsDataSelectedIndexingSolution = xsDataIntegrationInput.getSelectedIndexingSolution() self.__xsDataExperimentalConditionRefined = xsDataIntegrationInput.getExperimentalConditionRefined() # To be changed (see bug #40) if (self.__xsDataExperimentalConditionRefined is None): self.__xsDataExperimentalConditionRefined = xsDataSelectedIndexingSolution.getExperimentalConditionRefined() xsDataCollection = xsDataIntegrationInput.getDataCollection() xsDataSubWedgeList = xsDataCollection.getSubWedge() self.__edPluginIntegrationList = [] iIndex = 0 for xsDataSubWedge in xsDataSubWedgeList: iSubWedgeNumber = iIndex if (xsDataSubWedge.getSubWedgeNumber() is not None): # Use the incoming subwedge number if it exists iSubWedgeNumber = xsDataSubWedge.getSubWedgeNumber().getValue() edPluginIntegration = self.loadPlugin(self.__strPluginIntegrationName) if (not edPluginIntegration is None): iIndex += 1 xsDataIntegrationInputSubWedge = XSDataIntegrationInput() xsDataIntegrationInputSubWedge.setSelectedIndexingSolution(XSDataIndexingSolutionSelected.parseString(xsDataSelectedIndexingSolution.marshal())) xsDataIntegrationInputSubWedge.setExperimentalConditionRefined(XSDataExperimentalCondition.parseString(self.__xsDataExperimentalConditionRefined.marshal())) xsDataCollection = XSDataCollection() xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIntegrationInputSubWedge.setDataCollection(xsDataCollection) try: from EDHandlerXSDataMOSFLMv10 import EDHandlerXSDataMOSFLMv10 xsDataMOSFLMInputIntegration = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIntegration(xsDataIntegrationInputSubWedge) edPluginIntegration.setDataInput(xsDataMOSFLMInputIntegration) edPluginIntegration.setBaseName("%s-%02d" % (self.__strPluginIntegrationName, iIndex)) edPluginIntegration.connectSUCCESS(self.doSuccessActionIntegration) edPluginIntegration.connectFAILURE(self.doFailureActionIntegration) # Here we store the sub wedge number for use in the results self.__edPluginIntegrationList.append([iSubWedgeNumber, edPluginIntegration]) except Exception as strErrorMessage: self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() else: strErrorMessage = "EDPluginControlIntegrationv10.preProcess: could not load plugin %s" % self.__strPluginIntegrationName self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) self.setFailure()
def createDataInputCharacterisationFromDataSets(self, _pyListXSDataCCP4iDataSet): """ This method takes as input a list of ccp4i data sets. Each data set can contain several paths to image files. It runs the EDPluginControlSubWedgeAssemble plugin to read the experimental information from the image headers and then creates a single XSDataInputCharacterisation object. """ EDVerbose.DEBUG("EDPluginControlCCP4iv1_1.createDataInputCharacterisationFromDataSets") xsDataInputCharacterisation = None # We might have to run the plugin several times edActionCluster = EDActionCluster() pyListPluginControlSubWedgeAssemble = [] # Prepare the action cluster iIndex = 1 for xsDataCCP4iDataSet in _pyListXSDataCCP4iDataSet: edPluginControlSubWedgeAssemble = self.loadPlugin(self.__strPluginControlSubWedgeAssembleName, "SubWedgeAssemble-%02d" % iIndex) edPluginControlSubWedgeAssemble.connectSUCCESS(self.doSuccessActionSubWedgeAssemble) edPluginControlSubWedgeAssemble.connectFAILURE(self.doFailureActionSubWedgeAssemble) # Prepare the input for the sub wedge assemble plugin xsDataInputSubWedgeAssemble = XSDataInputSubWedgeAssemble() for xsDataImageFile in xsDataCCP4iDataSet.getImageFile(): xsDataInputSubWedgeAssemble.addFile(xsDataImageFile) edPluginControlSubWedgeAssemble.setDataInput(xsDataInputSubWedgeAssemble) pyListPluginControlSubWedgeAssemble.append(edPluginControlSubWedgeAssemble) self.addPluginToActionCluster(edPluginControlSubWedgeAssemble) iIndex += 1 # Run the action cluster synchronously self.executeActionCluster() self.synchronizeActionCluster() # Recuperate the output for edPluginControlSubWedgeAssemble in pyListPluginControlSubWedgeAssemble: if (edPluginControlSubWedgeAssemble.isFailure()): self.setFailure() else: xsDataResultSubWedgeAssemble = edPluginControlSubWedgeAssemble.getDataOutput() for xsDataSubWedge in xsDataResultSubWedgeAssemble.getSubWedge(): # Instantiate the xsDataInputCharacterisation object if it's not already done. xsDataCollection = None if (xsDataInputCharacterisation is None): xsDataInputCharacterisation = XSDataInputCharacterisation() else: xsDataCollection = xsDataInputCharacterisation.getDataCollection() if (xsDataCollection is None): xsDataCollection = XSDataCollection() xsDataCollection.addSubWedge(xsDataSubWedge) xsDataInputCharacterisation.setDataCollection(xsDataCollection) return xsDataInputCharacterisation
def getXSDataResultStrategy(self, _xsDataResultBest, _xsDataExperimentalCondition, _xsDataSample): xsDataResultStrategy = XSDataResultStrategy() #xsDataCollectionRunsBest = _xsDataResultBest.getCollectionRun() xsDataCollectionPlansBest = _xsDataResultBest.getCollectionPlan() for xsDataCollectionPlanBest in xsDataCollectionPlansBest: xsDataCollectionPlan = XSDataCollectionPlan() xsDataCollectionStrategy = XSDataCollection() xsDataDoubleTransmission = xsDataCollectionPlanBest.getStrategySummary().getAttenuation() for xsDataCollectionRunBest in xsDataCollectionPlanBest.getCollectionRun(): xsDataSubWedge = XSDataSubWedge() strXmlStringDataExperimentalCondition = _xsDataExperimentalCondition.marshal() xsDataExperimentalCondition = XSDataExperimentalCondition.parseString(strXmlStringDataExperimentalCondition) xsDataExperimentalCondition.getBeam().setExposureTime(xsDataCollectionRunBest.getExposureTime()) xsDataExperimentalCondition.getBeam().setTransmission(xsDataDoubleTransmission) xsDataExperimentalCondition.getDetector().setDistance(xsDataCollectionRunBest.getDistance()) xsDataExperimentalCondition.getGoniostat().setRotationAxisStart(xsDataCollectionRunBest.getPhiStart()) xsDataExperimentalCondition.getGoniostat().setOscillationWidth(xsDataCollectionRunBest.getPhiWidth()) fRotationAxisEnd = xsDataCollectionRunBest.getPhiStart().getValue() + xsDataCollectionRunBest.getNumberOfImages().getValue() * xsDataCollectionRunBest.getPhiWidth().getValue() xsDataExperimentalCondition.getGoniostat().setRotationAxisEnd(XSDataAngle(fRotationAxisEnd)) xsDataSubWedge.setExperimentalCondition(xsDataExperimentalCondition) xsDataSubWedge.setSubWedgeNumber(xsDataCollectionRunBest.getCollectionRunNumber()) xsDataCollectionStrategy.addSubWedge(xsDataSubWedge) xsDataCollectionStrategy.setSample(_xsDataSample) xsDataCollectionPlan.setCollectionStrategy(xsDataCollectionStrategy) xsDataStrategySummary = xsDataCollectionPlanBest.getStrategySummary() xsDataCollectionPlan.setStrategySummary(xsDataStrategySummary) xsDataStatistics = xsDataCollectionPlanBest.getStatisticalPrediction() xsDataCollectionPlan.setStatistics(xsDataStatistics) xsDataCollectionPlan.setCollectionPlanNumber(xsDataCollectionPlanBest.getCollectionPlanNumber()) xsDataResultStrategy.addCollectionPlan(xsDataCollectionPlan) return xsDataResultStrategy
def getXSDataStrategyResult(self, _xsDataBestOutput, _xsDataExperimentalCondition, _xsDataSample): xsDataStrategyResult = XSDataStrategyResult() #xsDataCollectionRunsBest = _xsDataBestOutput.getCollectionRun() xsDataCollectionPlansBest = _xsDataBestOutput.getCollectionPlan() for xsDataCollectionPlanBest in xsDataCollectionPlansBest: xsDataCollectionPlan = XSDataCollectionPlan() xsDataCollectionStrategy = XSDataCollection() for xsDataCollectionRunBest in xsDataCollectionPlanBest.getCollectionRun(): xsDataSubWedge = XSDataSubWedge() strXmlStringDataExperimentalCondition = _xsDataExperimentalCondition.marshal() xsDataExperimentalCondition = XSDataExperimentalCondition.parseString(strXmlStringDataExperimentalCondition) xsDataExperimentalCondition.getBeam().setExposureTime(xsDataCollectionRunBest.getExposureTime()) xsDataExperimentalCondition.getDetector().setDistance(xsDataCollectionRunBest.getDistance()) xsDataExperimentalCondition.getGoniostat().setRotationAxisStart(xsDataCollectionRunBest.getPhiStart()) xsDataExperimentalCondition.getGoniostat().setOscillationWidth(xsDataCollectionRunBest.getPhiWidth()) fRotationAxisEnd = xsDataCollectionRunBest.getPhiStart().getValue() + xsDataCollectionRunBest.getNumberOfImages().getValue() * xsDataCollectionRunBest.getPhiWidth().getValue() xsDataExperimentalCondition.getGoniostat().setRotationAxisEnd(XSDataAngle(fRotationAxisEnd)) xsDataSubWedge.setExperimentalCondition(xsDataExperimentalCondition) xsDataCollectionStrategy.addSubWedge(xsDataSubWedge) xsDataCollectionStrategy.setSample(_xsDataSample) xsDataCollectionPlan.setCollectionStrategy(xsDataCollectionStrategy) xsDataStrategySummary = xsDataCollectionPlanBest.getStrategySummary() xsDataCollectionPlan.setStrategySummary(xsDataStrategySummary) xsDataStatistics = xsDataCollectionPlanBest.getStatisticalPrediction() xsDataCollectionPlan.setStatistics(xsDataStatistics) xsDataStrategyResult.addCollectionPlan(xsDataCollectionPlan) return xsDataStrategyResult
def preProcess(self, _edObject=None): """ Gets the Configuration Parameters, if found, overrides default parameters """ EDPluginControl.preProcess(self, _edObject) EDVerbose.DEBUG("EDPluginControlGeneratePredictionv10.preProcess...") xsDataGeneratePredictionInput = self.getDataInput() xsDataSelectedIndexingSolution = xsDataGeneratePredictionInput.getSelectedIndexingSolution( ) xsDataExperimentalConditionRefined = xsDataSelectedIndexingSolution.getExperimentalConditionRefined( ) xsDataCollection = xsDataGeneratePredictionInput.getDataCollection() xsDataSubWedgeList = xsDataCollection.getSubWedge() # List containing instances of all the generate prediction plugins self.__listPluginGeneratePrediction = [] # Loop through all subwedges iIndex = 0 for xsDataSubWedge in xsDataSubWedgeList: xsDataImageList = xsDataSubWedge.getImage() # First find the lowest image number iLowestImageNumber = None for xsDataImage in xsDataImageList: iImageNumber = xsDataImage.getNumber().getValue() if (iLowestImageNumber is None): iLowestImageNumber = iImageNumber elif (iImageNumber < iLowestImageNumber): iLowestImageNumber = iImageNumber # Then loop through all images in a sub wedge for xsDataImage in xsDataImageList: iIndex += 1 edPluginGeneratePrediction = self.loadPlugin( self.__strPluginGeneratePredictionName, "%s-%02d" % (self.__strPluginGeneratePredictionName, iIndex)) xsDataGeneratePredictionInput = XSDataGeneratePredictionInput() xsDataGeneratePredictionInput.setSelectedIndexingSolution( XSDataIndexingSolutionSelected.parseString( xsDataSelectedIndexingSolution.marshal())) xsDataCollectionNew = XSDataCollection() xsDataSubWedgeNew = XSDataSubWedge() xsDataSubWedgeNew.addImage( XSDataImage.parseString(xsDataImage.marshal())) xsDataSubWedgeNew.setExperimentalCondition( XSDataExperimentalCondition.parseString( xsDataSubWedge.getExperimentalCondition().marshal())) # We must modify the rotationOscillationStart for the new subwedge xsDataGoniostatNew = xsDataSubWedgeNew.getExperimentalCondition( ).getGoniostat() fGoniostatRotationAxisStart = xsDataGoniostatNew.getRotationAxisStart( ).getValue() fGonioStatOscillationRange = xsDataGoniostatNew.getOscillationWidth( ).getValue() iImageNumber = xsDataImage.getNumber().getValue() fGoniostatRotationAxisStartNew = fGoniostatRotationAxisStart + ( iImageNumber - iLowestImageNumber) * fGonioStatOscillationRange xsDataGoniostatNew.setRotationAxisStart( XSDataAngle(fGoniostatRotationAxisStartNew)) # xsDataCollectionNew.addSubWedge(xsDataSubWedgeNew) xsDataGeneratePredictionInput.setDataCollection( xsDataCollectionNew) from EDHandlerXSDataMOSFLMv10 import EDHandlerXSDataMOSFLMv10 xsDataMOSFLMInputGeneratePrediction = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputGeneratePrediction( xsDataGeneratePredictionInput) edPluginGeneratePrediction.setDataInput( xsDataMOSFLMInputGeneratePrediction) self.__listPluginGeneratePrediction.append( edPluginGeneratePrediction)
class EDPluginControlImageQualityIndicatorsv1_3(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__(self): EDPluginControl.__init__(self) self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1" self.strPluginName = "EDPluginDistlSignalStrengthv1_1" self.strPluginNameThinClient = "EDPluginDistlSignalStrengthThinClientv1_1" self.strPluginNameControlBackground3D = "EDPluginControlBackground3Dv1_0" self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_4" self.strIndexingMOSFLMPluginName = "EDPluginMOSFLMIndexingv10" self.edPluginMOSFLMIndexing = None self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.edPluginReadImageHeader = None self.edPluginControlBackground3D = None self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = [] self.listPluginControlBackground3D = [] self.xsDataResultControlImageQualityIndicators = None self.edPluginMXWaitFile = None # Default time out for wait file self.fMXWaitFileTimeOut = 30 # s # Flag for using the thin client - disabled as of 2016/07/20 self.bUseThinClient = False self.edPluginISPyB = None self.listPluginMOSFLM = [] self.defaultMinImageSize = 1000000 self.minImageSize = None def checkParameters(self): """ Checks the mandatory parameters """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_3.checkParameters") self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlReadImageHeaderv10.configure") self.fMXWaitFileTimeOut = float( self.config.get("MXWaitFileTimeOut", self.fMXWaitFileTimeOut)) self.minImageSize = self.config.get("minImageSize") if self.minImageSize is None: self.minImageSize = self.defaultMinImageSize def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_3.process") EDUtilsParallel.initializeNbThread() # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPlugin = [] for xsDataImage in listXSDataImage: self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(5000000)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): strError = "Time-out while waiting for image %s" % xsDataImage.path.value self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() edPluginControlBackground3D = self.loadPlugin( self.strPluginNameControlBackground3D) listPlugin.append([ edPluginPluginExecImageQualityIndicator, edPluginControlBackground3D ]) xsDataInputControlBackground3D = XSDataInputControlBackground3D( ) xsDataInputControlBackground3D.addImage( XSDataFile(xsDataImage.path)) edPluginControlBackground3D.dataInput = xsDataInputControlBackground3D edPluginControlBackground3D.execute() listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for pluginPair in listPlugin: edPluginPluginExecImageQualityIndicator = pluginPair[0] edPluginControlBackground3D = pluginPair[1] edPluginPluginExecImageQualityIndicator.synchronize() edPluginControlBackground3D.synchronize() xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) if edPluginControlBackground3D.dataOutput.imageBackground != []: xsDataImageQualityIndicators.background3D_estimate = edPluginControlBackground3D.dataOutput.imageBackground[ 0].estimate self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have background3D_estimate from all images: has_background3D_estimate = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.background3D_estimate is None: has_background3D_estimate = False if has_background3D_estimate: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators. background3D_estimate.value) else: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators. totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection( self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing( xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin( self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([ edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator ]) edPluginMOSFLMIndexing.setDataInput( xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult( xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution # print xsDataIndexingResult.marshal() # xsDataResultISPyB = edPluginISPyB.dataOutput # if xsDataResultISPyB is not None: # print xsDataResultISPyB.marshal() def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) if self.edPluginISPyB is not None: # Synchronize ISPyB plugin self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_3.finallyProcess") self.edPluginISPyB.synchronize() listId = [] for xsDataInteger in self.edPluginISPyB.dataOutput.imageQualityIndicatorsId: listId.append(xsDataInteger.value) self.DEBUG("ISPyB imageQualityIndicatorIds = %r" % listId) self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def generateExecutiveSummary(self, _edPlugin=None): self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_3.generateExecutiveSummary" ) self.addErrorWarningMessagesToExecutiveSummary( "Image quality indicator plugin execution failure! Error messages: " ) self.addExecutiveSummaryLine( "Summary of image quality indicators with %s :" % self.strPluginName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary( edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False)
def getXSDataResultStrategy(self, _xsDataResultBest, _xsDataExperimentalCondition, _xsDataSample): xsDataResultStrategy = XSDataResultStrategy() listXSDataBestCollectionPlan = _xsDataResultBest.getCollectionPlan() for xsDataBestCollectionPlan in listXSDataBestCollectionPlan: xsDataCollectionPlan = XSDataCollectionPlan() xsDataCollectionStrategy = XSDataCollection() xsDataBestStrategySummary = xsDataBestCollectionPlan.getStrategySummary( ) xsDataDoubleTransmission = xsDataBestStrategySummary.getTransmission( ) for xsDataBestCollectionRun in xsDataBestCollectionPlan.getCollectionRun( ): xsDataSubWedge = XSDataSubWedge() strXmlStringDataExperimentalCondition = _xsDataExperimentalCondition.marshal( ) xsDataExperimentalCondition = XSDataExperimentalCondition.parseString( strXmlStringDataExperimentalCondition) xsDataExperimentalCondition.getBeam().setExposureTime( xsDataBestCollectionRun.getExposureTime()) if (xsDataBestCollectionRun.getTransmission() is None): xsDataExperimentalCondition.getBeam().setTransmission( xsDataDoubleTransmission) else: xsDataExperimentalCondition.getBeam().setTransmission( xsDataBestCollectionRun.getTransmission()) xsDataExperimentalCondition.getDetector().setDistance( xsDataBestStrategySummary.getDistance()) xsDataExperimentalCondition.getGoniostat( ).setRotationAxisStart(xsDataBestCollectionRun.getPhiStart()) xsDataExperimentalCondition.getGoniostat().setOscillationWidth( xsDataBestCollectionRun.getPhiWidth()) fRotationAxisEnd = xsDataBestCollectionRun.getPhiStart( ).getValue( ) + xsDataBestCollectionRun.getNumberOfImages().getValue( ) * xsDataBestCollectionRun.getPhiWidth().getValue() xsDataExperimentalCondition.getGoniostat().setRotationAxisEnd( XSDataAngle(fRotationAxisEnd)) xsDataSubWedge.setExperimentalCondition( xsDataExperimentalCondition) xsDataSubWedge.setSubWedgeNumber( xsDataBestCollectionRun.getCollectionRunNumber()) if xsDataBestCollectionRun.getCrystalPosition(): xsDataSubWedge.setAction( XSDataString("Crystal position: %d" % xsDataBestCollectionRun. getCrystalPosition().getValue())) else: xsDataSubWedge.setAction( xsDataBestCollectionRun.getAction()) xsDataCollectionStrategy.addSubWedge(xsDataSubWedge) xsDataCollectionStrategy.setSample(_xsDataSample) xsDataCollectionPlan.setCollectionStrategy( xsDataCollectionStrategy) xsDataStrategySummary = XSDataStrategySummary() xsDataStrategySummary.setCompleteness( xsDataBestStrategySummary.getCompleteness()) xsDataStrategySummary.setISigma( xsDataBestStrategySummary.getISigma()) xsDataStrategySummary.setRankingResolution( xsDataBestStrategySummary.getRankingResolution()) xsDataStrategySummary.setRedundancy( xsDataBestStrategySummary.getRedundancy()) xsDataStrategySummary.setResolution( xsDataBestStrategySummary.getResolution()) xsDataStrategySummary.setResolutionReasoning( xsDataBestStrategySummary.getResolutionReasoning()) xsDataStrategySummary.setTotalDataCollectionTime( xsDataBestStrategySummary.getTotalDataCollectionTime()) xsDataStrategySummary.setTotalExposureTime( xsDataBestStrategySummary.getTotalExposureTime()) xsDataCollectionPlan.setStrategySummary(xsDataStrategySummary) if xsDataBestCollectionPlan.getStatisticalPrediction() is not None: xsDataStatisticsStrategy = XSDataStatisticsStrategy.parseString( xsDataBestCollectionPlan.getStatisticalPrediction( ).marshal()) xsDataCollectionPlan.setStatistics(xsDataStatisticsStrategy) xsDataCollectionPlan.setCollectionPlanNumber( xsDataBestCollectionPlan.getCollectionPlanNumber()) xsDataResultStrategy.addCollectionPlan(xsDataCollectionPlan) if _xsDataResultBest.getPathToLogFile() != None: xsDataResultStrategy.setBestLogFile( _xsDataResultBest.getPathToLogFile()) return xsDataResultStrategy
class EDPluginControlImageQualityIndicatorsv1_2(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__(self): EDPluginControl.__init__(self) self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1" self.strPluginName = "EDPluginDistlSignalStrengthv1_1" self.strPluginNameThinClient = "EDPluginDistlSignalStrengthv1_1" self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_4" self.strIndexingMOSFLMPluginName = "EDPluginMOSFLMIndexingv10" self.edPluginMOSFLMIndexing = None self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.edPluginReadImageHeader = None self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = [] self.xsDataResultControlImageQualityIndicators = None self.edPluginMXWaitFile = None # Default time out for wait file self.fMXWaitFileTimeOut = 30 #s # Flag for using the thin client self.bUseThinClient = True self.edPluginISPyB = None self.listPluginMOSFLM = [] self.bDoISPyBUpload = True self.defaultMinImageSize = 1000000 self.minImageSize = None def checkParameters(self): """ Checks the mandatory parameters """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.checkParameters") self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlReadImageHeaderv10.configure") self.fMXWaitFileTimeOut = float( self.config.get("MXWaitFileTimeOut", self.fMXWaitFileTimeOut)) self.bDoISPyBUpload = self.config.get("do_ispyb_upload") if self.bDoISPyBUpload is None or self.bDoISPyBUpload == "false": self.bDoISPyBUpload = False self.minImageSize = self.config.get("minImageSize") if self.minImageSize is None: self.minImageSize = self.defaultMinImageSize def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.process") EDUtilsParallel.initializeNbThread() # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPlugin = [] for xsDataImage in listXSDataImage: self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): strError = "Time-out while waiting for image %s" % xsDataImage.path.value self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) listPlugin.append(edPluginPluginExecImageQualityIndicator) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for edPluginPluginExecImageQualityIndicator in listPlugin: edPluginPluginExecImageQualityIndicator.synchronize() xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString( \ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.bDoISPyBUpload: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key= lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value ) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection( self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing( xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin( self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([ edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator ]) edPluginMOSFLMIndexing.setDataInput( xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult( xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution # print xsDataIndexingResult.marshal() # xsDataResultISPyB = edPluginISPyB.dataOutput # if xsDataResultISPyB is not None: #print xsDataResultISPyB.marshal() def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) # Synchronize ISPyB plugin self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.finallyProcess") if self.bDoISPyBUpload: self.edPluginISPyB.synchronize() listId = [] for xsDataInteger in self.edPluginISPyB.dataOutput.imageQualityIndicatorsId: listId.append(xsDataInteger.value) self.DEBUG("ISPyB imageQualityIndicatorIds = %r" % listId) self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def generateExecutiveSummary(self, _edPlugin=None): self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_2.generateExecutiveSummary" ) self.addErrorWarningMessagesToExecutiveSummary( "Image quality indicator plugin execution failure! Error messages: " ) self.addExecutiveSummaryLine( "Summary of image quality indicators with %s :" % self.strPluginName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary( edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False)
def preProcess(self, _edObject=None): """ Gets the Configuration Parameters, if found, overrides default parameters """ EDPluginControl.preProcess(self, _edObject) self.DEBUG("EDPluginControlIntegrationv10.preProcess...") xsDataIntegrationInput = self.getDataInput() xsDataSelectedIndexingSolution = xsDataIntegrationInput.getSelectedIndexingSolution( ) self.__xsDataExperimentalConditionRefined = xsDataIntegrationInput.getExperimentalConditionRefined( ) # To be changed (see bug #40) if (self.__xsDataExperimentalConditionRefined is None): self.__xsDataExperimentalConditionRefined = xsDataSelectedIndexingSolution.getExperimentalConditionRefined( ) xsDataCollection = xsDataIntegrationInput.getDataCollection() xsDataSubWedgeList = xsDataCollection.getSubWedge() self.__edPluginIntegrationList = [] iIndex = 0 for xsDataSubWedge in xsDataSubWedgeList: iSubWedgeNumber = iIndex if (xsDataSubWedge.getSubWedgeNumber() is not None): # Use the incoming subwedge number if it exists iSubWedgeNumber = xsDataSubWedge.getSubWedgeNumber().getValue() edPluginIntegration = self.loadPlugin( self.__strPluginIntegrationName) if (not edPluginIntegration is None): iIndex += 1 xsDataIntegrationInputSubWedge = XSDataIntegrationInput() xsDataIntegrationInputSubWedge.setSelectedIndexingSolution( XSDataIndexingSolutionSelected.parseString( xsDataSelectedIndexingSolution.marshal())) xsDataIntegrationInputSubWedge.setExperimentalConditionRefined( XSDataExperimentalCondition.parseString( self.__xsDataExperimentalConditionRefined.marshal())) xsDataCollection = XSDataCollection() xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIntegrationInputSubWedge.setDataCollection( xsDataCollection) try: from EDHandlerXSDataMOSFLMv10 import EDHandlerXSDataMOSFLMv10 xsDataMOSFLMInputIntegration = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIntegration( xsDataIntegrationInputSubWedge) edPluginIntegration.setDataInput( xsDataMOSFLMInputIntegration) edPluginIntegration.setBaseName( "%s-%02d" % (self.__strPluginIntegrationName, iIndex)) edPluginIntegration.connectSUCCESS( self.doSuccessActionIntegration) edPluginIntegration.connectFAILURE( self.doFailureActionIntegration) # Here we store the sub wedge number for use in the results self.__edPluginIntegrationList.append( [iSubWedgeNumber, edPluginIntegration]) except Exception as strErrorMessage: self.addErrorMessage(strErrorMessage) self.ERROR(strErrorMessage) self.setFailure() else: strErrorMessage = "EDPluginControlIntegrationv10.preProcess: could not load plugin %s" % self.__strPluginIntegrationName self.error(strErrorMessage) self.addErrorMessage(strErrorMessage) self.setFailure()
class EDPluginControlGridScreeningv1_0(EDPluginControl): """ This plugin is a "light-weight" characterisation to be used for processing images from grid scans. """ def __init__(self): EDPluginControl.__init__(self) self.setXSDataInputClass(XSDataInputGridScreening) self.strControlReadImageHeaderPluginName = "EDPluginControlReadImageHeaderv10" self.edPluginControlReadImageHeader = None self.strControlledIndicatorsPluginName = "EDPluginControlImageQualityIndicatorsv1_4" self.edPluginControlIndicators = None self.strISPyBStoreImageQualityIndicatorsPluginName = "EDPluginISPyBStoreImageQualityIndicatorsv1_4" self.edPluginISPyBStoreImageQualityIndicators = None self.strIndexingMOSFLMPluginName = "EDPluginMOSFLMIndexingv10" self.edPluginMOSFLMIndexing = None self.strPluginControlIntegration = "EDPluginControlIntegrationv10" self.edPluginControlIntegration = None self.strPluginControlStrategy = "EDPluginControlStrategyv1_2" self.edPluginControlStrategy = None self.strPluginExecMtz2Various = "EDPluginExecMtz2Variousv1_0" self.edPluginExecMtz2Various = None self.strImageFile = None self.xsDataIndexingResultMOSFLM = None self.xsDataCrystal = None self.strCharacterisationShortSummary = "" self.strStatusMessage = "" self.xsDataDiffractionPlan = None self.xsDataCollection = None self.xsDataIndexingResult = None self.xsDataStrategyResult = None self.xsDataImageQualityIndicators = None self.bStoreImageQualityIndicatorsInISPyB = False self.bDoOnlyImageQualityIndicators = False self.bDoOnlyIntegrationWithXMLOutput = False self.iImageQualityIndicatorsId = None self.xsDataGridScreeningResultIntegration = None def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginControlGridScreeningv1_0.checkParameters") self.checkMandatoryParameters(self.getDataInput(), "Data Input is None") self.checkMandatoryParameters(self.getDataInput().getImageFile(), "imageFile") def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlGridScreeningv1_0.preProcess") # Load the plugins self.edPluginControlReadImageHeader = self.loadPlugin(self.strControlReadImageHeaderPluginName, \ "ReadImageHeader") self.edPluginControlIndicators = self.loadPlugin(self.strControlledIndicatorsPluginName, \ "ControlIndicators") self.edPluginMOSFLMIndexing = self.loadPlugin(self.strIndexingMOSFLMPluginName, \ "IndexingMOSFLM") self.edPluginControlIntegration = self.loadPlugin(self.strPluginControlIntegration, \ "Integration") self.edPluginControlStrategy = self.loadPlugin(self.strPluginControlStrategy, \ "Strategy") self.edPluginExecMtz2Various = self.loadPlugin(self.strPluginExecMtz2Various, \ "Mtz2Various") # Input data self.strImageFile = self.getDataInput().getImageFile().getPath().getValue() self.xsDataGridScreeningFileNameParameters = self.getFileNameParameters(self.strImageFile) self.xsDataDiffractionPlan = self.getDataInput().getDiffractionPlan() if self.xsDataDiffractionPlan is None: self.xsDataDiffractionPlan = XSDataDiffractionPlan() if self.xsDataDiffractionPlan.getMaxExposureTimePerDataCollection() is None: # Default max esposure time: 10000s self.xsDataDiffractionPlan.setMaxExposureTimePerDataCollection(XSDataTime(10000)) self.xsDataDiffractionPlan.setEstimateRadiationDamage(XSDataBoolean(False)) # Image quality indicators if self.getDataInput().getStoreImageQualityIndicatorsInISPyB(): self.bStoreImageQualityIndicatorsInISPyB = self.getDataInput().getStoreImageQualityIndicatorsInISPyB().getValue() if self.getDataInput().getDoOnlyImageQualityIndicators(): self.bDoOnlyImageQualityIndicators = self.getDataInput().getDoOnlyImageQualityIndicators().getValue() if self.getDataInput().getDoOnlyIntegrationWithXMLOutput(): self.bDoOnlyIntegrationWithXMLOutput = self.getDataInput().getDoOnlyIntegrationWithXMLOutput().getValue() if self.bStoreImageQualityIndicatorsInISPyB: self.edPluginISPyBStoreImageQualityIndicators = self.loadPlugin(self.strISPyBStoreImageQualityIndicatorsPluginName, \ "ISPyBStoreImageQualityIndicators") def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlGridScreeningv1_0.process") xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.setImage(image=XSDataFile(path=XSDataString(self.strImageFile))) self.edPluginControlReadImageHeader.setDataInput(xsDataInputReadImageHeader) self.edPluginControlReadImageHeader.connectSUCCESS(self.doSuccessReadImageHeader) self.edPluginControlReadImageHeader.connectFAILURE(self.doFailureReadImageHeader) self.executePluginSynchronous(self.edPluginControlReadImageHeader) def finallyProcess(self, _edObject=None): EDPluginControl.finallyProcess(self) self.DEBUG("EDPluginControlGridScreeningv1_0.finallyProcess") # Synchronise if necessary if self.edPluginISPyBStoreImageQualityIndicators is not None: self.edPluginISPyBStoreImageQualityIndicators.synchronize() # Build up the output object strComment = "" xsDataResultGridScreening = XSDataResultGridScreening() if self.xsDataGridScreeningFileNameParameters is not None: xsDataResultGridScreening.setFileNameParameters(self.xsDataGridScreeningFileNameParameters) if self.xsDataImageQualityIndicators is None: strComment = "No image quality indicators" else: xsDataResultGridScreening.setImageQualityIndicators(self.xsDataImageQualityIndicators) if self.xsDataImageQualityIndicators.getIceRings() is not None: if self.xsDataImageQualityIndicators.getIceRings().getValue() > 1: strComment = "Ice rings detected" if self.xsDataIndexingResult is None: if strComment == "": strComment = "No indexing result" else: strComment += ", no indexing result" else: xsDataSelectedSolution = self.xsDataIndexingResult.getSelectedSolution() xsDataResultGridScreening.setMosaicity(xsDataSelectedSolution.getCrystal().getMosaicity()) if self.xsDataStrategyResult is None: if strComment == "": strComment = "No strategy result" else: strComment += ", no strategy result" else: xsDataCollectionPlan = self.xsDataStrategyResult.getCollectionPlan()[0] xsDataStrategySummary = xsDataCollectionPlan.getStrategySummary() xsDataResultGridScreening.setRankingResolution(xsDataStrategySummary.getRankingResolution()) xsDataResultGridScreening.setResultIntegration(self.xsDataGridScreeningResultIntegration) xsDataResultGridScreening.setComment(XSDataString(strComment)) if self.iImageQualityIndicatorsId is not None: xsDataResultGridScreening.setImageQualityIndicatorsId(XSDataInteger(self.iImageQualityIndicatorsId)) # print xsDataResultGridScreening.marshal() self.setDataOutput(xsDataResultGridScreening) def doSuccessReadImageHeader(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doSuccessReadImageHeader") self.retrieveSuccessMessages(_edPlugin, "EDPluginControlGridScreeningv1_0.doSuccessReadImageHeader") xsDataResultReadImageHeader = self.edPluginControlReadImageHeader.getDataOutput() if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.getSubWedge() self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) self.xsDataCollection.setDiffractionPlan(self.xsDataDiffractionPlan) if not self.bDoOnlyIntegrationWithXMLOutput: xsDataInputControlImageQualityIndicators = XSDataInputControlImageQualityIndicators() if self.bStoreImageQualityIndicatorsInISPyB: xsDataInputControlImageQualityIndicators.doUploadToIspyb = XSDataBoolean(True) else: xsDataInputControlImageQualityIndicators.doUploadToIspyb = XSDataBoolean(False) xsDataInputControlImageQualityIndicators.addImage(XSDataImage(path=XSDataString(self.strImageFile))) self.edPluginControlIndicators.setDataInput(xsDataInputControlImageQualityIndicators) self.edPluginControlIndicators.connectSUCCESS(self.doSuccessIndicators) self.edPluginControlIndicators.connectFAILURE(self.doFailureIndicators) self.executePluginSynchronous(self.edPluginControlIndicators) else: xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(self.xsDataCollection) from EDHandlerXSDataMOSFLMv10 import EDHandlerXSDataMOSFLMv10 xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(xsDataIndexingInput) self.edPluginMOSFLMIndexing.connectSUCCESS(self.doSuccessIndexingMOSFLM) self.edPluginMOSFLMIndexing.connectFAILURE(self.doFailureIndexingMOSFLM) self.edPluginMOSFLMIndexing.setDataInput(xsDataMOSFLMIndexingInput) self.edPluginMOSFLMIndexing.executeSynchronous() def doFailureReadImageHeader(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doFailureReadImageHeader") self.retrieveFailureMessages(_edPlugin, "EDPluginControlGridScreeningv1_0.doSuccessReadImageHeader") def doSuccessIndicators(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doSuccessIndicators") # self.retrieveSuccessMessages(_edPlugin, "EDPluginControlGridScreeningv1_0.doSuccessIndexingIndicators") if self.edPluginControlIndicators.hasDataOutput(): self.xsDataImageQualityIndicators = self.edPluginControlIndicators.getDataOutput().getImageQualityIndicators()[0] # Continue only if requested if not self.bDoOnlyImageQualityIndicators: xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(self.xsDataCollection) from EDHandlerXSDataMOSFLMv10 import EDHandlerXSDataMOSFLMv10 xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(xsDataIndexingInput) self.edPluginMOSFLMIndexing.connectSUCCESS(self.doSuccessIndexingMOSFLM) self.edPluginMOSFLMIndexing.connectFAILURE(self.doFailureIndexingMOSFLM) self.edPluginMOSFLMIndexing.setDataInput(xsDataMOSFLMIndexingInput) self.edPluginMOSFLMIndexing.executeSynchronous() def doFailureIndicators(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doFailureIndicators") strErrorMessage = "Execution of Indexing and Indicators plugin failed. Execution of characterisation aborted." self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) def doSuccessISPyBStoreImageQualityIndicators(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doSuccessISPyBStoreImageQualityIndicators") self.retrieveSuccessMessages(_edPlugin, "EDPluginControlGridScreeningv1_0.doSuccessISPyBStoreImageQualityIndicators") self.iImageQualityIndicatorsId = self.edPluginISPyBStoreImageQualityIndicators.getDataOutput().getImageQualityIndicatorsId().getValue() if self.iImageQualityIndicatorsId is None: self.WARNING("Image quality indicators result not stored in ISPyB!") def doFailureISPyBStoreImageQualityIndicators(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doFailureISPyBStoreImageQualityIndicators") strErrorMessage = "Execution of store ISPyB image quality indicators plugin failed." self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) def doSuccessIndexingMOSFLM(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doSuccessIndexingMOSFLM") self.retrieveSuccessMessages(_edPlugin, "EDPluginControlGridScreeningv1_0.doIntegrationToStrategyTransition") xsDataMOSFLMIndexingOutput = self.edPluginMOSFLMIndexing.getDataOutput() xsDataExperimentalConditionRefined = None if self.hasDataInput("refinedExperimentalCondition"): xsDataExperimentalConditionRefined = self.getDataInput("refinedExperimentalCondition")[0] else: # Use the experimental condition from the xsDataCollection xsDataExperimentalConditionRefined = self.xsDataCollection.getSubWedge()[0].getExperimentalCondition() from EDHandlerXSDataMOSFLMv10 import EDHandlerXSDataMOSFLMv10 self.xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(xsDataMOSFLMIndexingOutput, xsDataExperimentalConditionRefined) xsDataIntegrationInput = XSDataIntegrationInput() xsDataIntegrationInput.setDataCollection(self.xsDataCollection) xsDataIntegrationInput.setExperimentalConditionRefined(self.xsDataIndexingResult.getSelectedSolution().getExperimentalConditionRefined()) xsDataIntegrationInput.setSelectedIndexingSolution(self.xsDataIndexingResult.getSelectedSolution()) self.edPluginControlIntegration.connectSUCCESS(self.doSuccessIntegration) self.edPluginControlIntegration.connectFAILURE(self.doFailureIntegration) self.edPluginControlIntegration.setDataInput(xsDataIntegrationInput) self.executePluginSynchronous(self.edPluginControlIntegration) def doFailureIndexingMOSFLM(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doFailureActionIndexing") self.retrieveFailureMessages(_edPlugin, "EDPluginControlCharacterisationv1_1.doFailureActionIndexing") def doSuccessIntegration(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doSuccessIntegration") self.retrieveSuccessMessages(_edPlugin, "EDPluginControlGridScreeningv1_0.doSuccessIntegration") self.addStatusMessage("Integration successful.") self.xsDataIntegrationOutput = self.edPluginControlIntegration.getDataOutput() # Integration short summary if self.edPluginControlIntegration.hasDataOutput("integrationShortSummary"): self.strCharacterisationShortSummary += self.edPluginControlIntegration.getDataOutput("integrationShortSummary")[0].getValue() # self.DEBUG( self.xsDataExperimentCharacterisation.marshal() ) if self.bDoOnlyIntegrationWithXMLOutput: # Run mtz2various xsDataInputMtz2Various = XSDataInputMtz2Various() xsDataInputMtz2Various.setMtzfile(self.edPluginControlIntegration.getDataOutput().getIntegrationSubWedgeResult()[0].getGeneratedMTZFile()) xsDataInputMtz2Various.addLabin(XSDataString("I=I")) xsDataInputMtz2Various.addLabin(XSDataString("SIGI=SIGI")) xsDataInputMtz2Various.setOutput(XSDataString("USER '(3I4,2F10.1)'")) self.edPluginExecMtz2Various.setDataInput(xsDataInputMtz2Various) self.edPluginExecMtz2Various.executeSynchronous() strHklFilePath = self.edPluginExecMtz2Various.getDataOutput().getHklfile().getPath().getValue() strIntegration = EDUtilsFile.readFile(strHklFilePath) # Output the result in XML format self.xsDataGridScreeningResultIntegration = XSDataGridScreeningResultIntegration() self.xsDataGridScreeningResultIntegration.setFileName(os.path.basename(self.strImageFile)) self.xsDataGridScreeningResultIntegration.setFileDirectory(os.path.dirname(self.strImageFile)) self.xsDataGridScreeningResultIntegration.setIntegratedData(strIntegration) else: # We continue with the strategy calculation xsDataInputStrategy = XSDataInputStrategy() xsDataSolutionSelected = self.xsDataIndexingResult.getSelectedSolution() xsDataInputStrategy.setCrystalRefined(xsDataSolutionSelected.getCrystal()) xsDataInputStrategy.setSample(self.xsDataCollection.getSample()) xsDataIntegrationSubWedgeResultList = self.xsDataIntegrationOutput.getIntegrationSubWedgeResult() xsDataInputStrategy.setBestFileContentDat(xsDataIntegrationSubWedgeResultList[0].getBestfileDat()) xsDataInputStrategy.setBestFileContentPar(xsDataIntegrationSubWedgeResultList[0].getBestfilePar()) xsDataInputStrategy.setExperimentalCondition(xsDataIntegrationSubWedgeResultList[0].getExperimentalConditionRefined()) xsDataInputStrategy.setDataCollection(self.xsDataCollection) for xsDataIntegrationSubWedgeResult in xsDataIntegrationSubWedgeResultList: xsDataInputStrategy.addBestFileContentHKL(xsDataIntegrationSubWedgeResult.getBestfileHKL()) xsDataInputStrategy.setDiffractionPlan(self.xsDataDiffractionPlan) self.edPluginControlStrategy.connectSUCCESS(self.doSuccessStrategy) self.edPluginControlStrategy.connectFAILURE(self.doFailureStrategy) self.edPluginControlStrategy.setDataInput(xsDataInputStrategy) self.executePluginSynchronous(self.edPluginControlStrategy) def doFailureIntegration(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doFailureIntegration") strErrorMessage = "Execution of integration plugin failed." self.addStatusMessage("Integration FAILURE.") self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) def doSuccessStrategy(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doSuccessStrategy") self.retrieveSuccessMessages(self.edPluginControlStrategy, "EDPluginControlGridScreeningv1_0.doSuccessStrategy") self.xsDataStrategyResult = self.edPluginControlStrategy.getDataOutput() def doFailureStrategy(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doFailureStrategy") self.retrieveFailureMessages(self.edPluginControlStrategy, "EDPluginControlGridScreeningv1_0.doFailureStrategy") strErrorMessage = "Strategy calculation FAILURE." self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) # def generateExecutiveSummary(self, _edPlugin): # """ # Generates a summary of the execution of the plugin. # """ # self.DEBUG("EDPluginControlGridScreeningv1_0.generateExecutiveSummary") # self.addExecutiveSummaryLine("Summary of characterisation:") # xsDataInputStrategy = self.getDataInput() # xsDataCollection = xsDataInputStrategy.getDataCollection() # xsDataDiffractionPlan = xsDataCollection.getDiffractionPlan() # self.addExecutiveSummaryLine("Diffraction plan:") # if (xsDataDiffractionPlan.getComplexity() is not None): # self.addExecutiveSummaryLine("BEST complexity : %s" % xsDataDiffractionPlan.getComplexity().getValue()) # if (xsDataDiffractionPlan.getAimedCompleteness() is not None): # self.addExecutiveSummaryLine("Aimed completeness : %6.1f [%%]" % (100.0 * xsDataDiffractionPlan.getAimedCompleteness().getValue())) # if (xsDataDiffractionPlan.getRequiredCompleteness() is not None): # self.addExecutiveSummaryLine("Required completeness : %6.1f [%%]" % (100.0 * xsDataDiffractionPlan.getRequiredCompleteness().getValue())) # if (xsDataDiffractionPlan.getAimedIOverSigmaAtHighestResolution() is not None): # self.addExecutiveSummaryLine("Aimed I/sigma at highest resolution : %6.1f" % xsDataDiffractionPlan.getAimedIOverSigmaAtHighestResolution().getValue()) # if (xsDataDiffractionPlan.getAimedResolution() is not None): # self.addExecutiveSummaryLine("Aimed resolution : %6.1f [A]" % xsDataDiffractionPlan.getAimedResolution().getValue()) # if (xsDataDiffractionPlan.getRequiredResolution() is not None): # self.addExecutiveSummaryLine("Required resolution : %6.1f [A]" % xsDataDiffractionPlan.getRequiredResolution().getValue()) # if (xsDataDiffractionPlan.getAimedMultiplicity() is not None): # self.addExecutiveSummaryLine("Aimed multiplicity : %6.1f" % xsDataDiffractionPlan.getAimedMultiplicity().getValue()) # if (xsDataDiffractionPlan.getRequiredMultiplicity() is not None): # self.addExecutiveSummaryLine("Required multiplicity : %6.1f" % xsDataDiffractionPlan.getRequiredMultiplicity().getValue()) # if (xsDataDiffractionPlan.getForcedSpaceGroup() is not None): # self.addExecutiveSummaryLine("Forced space group : %6s" % xsDataDiffractionPlan.getForcedSpaceGroup().getValue()) # if (xsDataDiffractionPlan.getMaxExposureTimePerDataCollection() is not None): # self.addExecutiveSummaryLine("Max exposure time per data collection : %6.1f [s]" % xsDataDiffractionPlan.getMaxExposureTimePerDataCollection().getValue()) # self.addExecutiveSummarySeparator() # if self.edPluginControlIndexingIndicators is not None: # self.appendExecutiveSummary(self.edPluginControlIndexingIndicators, "") # if self.edPluginControlIndexingLabelit is not None: # self.appendExecutiveSummary(self.edPluginControlIndexingLabelit, "") # if self.edPluginControlIntegration is not None: # self.appendExecutiveSummary(self.edPluginControlIntegration, "") # if self.edPluginControlStrategy is not None: # self.appendExecutiveSummary(self.edPluginControlStrategy, "") # self.addExecutiveSummarySeparator() # if self.strCharacterisationShortSummary is not None: # self.addExecutiveSummaryLine("Characterisation short summary:") # self.addExecutiveSummaryLine("") # if self.strStatusMessage != None: # for strLine in self.strStatusMessage.split(". "): # if strLine.endswith("."): # self.addExecutiveSummaryLine(strLine) # else: # self.addExecutiveSummaryLine(strLine + ".") # self.addExecutiveSummaryLine("") # for strLine in self.strCharacterisationShortSummary.split("\n"): # if strLine != "\n": # self.addExecutiveSummaryLine(strLine) # self.addErrorWarningMessagesToExecutiveSummary("Characterisation error and warning messages: ") # self.addExecutiveSummarySeparator() def addStatusMessage(self, _strStatusMessage): if self.strStatusMessage != "": self.strStatusMessage += " " self.strStatusMessage += _strStatusMessage def getFileNameParameters(self, _strFileName): """Method for extracting the rotation start angle, the two motor positions and the grid scan image no from the file name""" # Typical file name: mesh_0_21.676_-0.051_22_001.mccd listParts = os.path.basename(_strFileName).split("_") xsDataGridScreeningFileNameParameters = XSDataGridScreeningFileNameParameters() try: strScanId1 = listParts[1] xsDataGridScreeningFileNameParameters.setScanId1(XSDataString(strScanId1)) strMotorPosition1 = listParts[2] xsDataGridScreeningFileNameParameters.setMotorPosition1(XSDataString(strMotorPosition1)) strMotorPosition2 = listParts[3] xsDataGridScreeningFileNameParameters.setMotorPosition2(XSDataString(strMotorPosition2)) strScanId2 = listParts[4] xsDataGridScreeningFileNameParameters.setScanId2(XSDataString(strScanId2)) except: xsDataGridScreeningFileNameParameters = None return xsDataGridScreeningFileNameParameters
class EDPluginControlImageQualityIndicatorsv1_4(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__ (self): EDPluginControl.__init__(self) self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1" self.strPluginName = "EDPluginDistlSignalStrengthv1_1" self.strPluginNameThinClient = "EDPluginDistlSignalStrengthThinClientv1_1" self.strPluginNameControlDozor = "EDPluginControlDozorv1_0" self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_4" self.strIndexingMOSFLMPluginName = "EDPluginMOSFLMIndexingv10" self.edPluginMOSFLMIndexing = None self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.edPluginReadImageHeader = None self.edPluginControlDozor = None self.strPluginControlH5ToCBF = "EDPluginControlH5ToCBFv1_1" self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = [] self.listPluginControlDozor = [] self.xsDataResultControlImageQualityIndicators = None self.edPluginMXWaitFile = None # Default time out for wait file self.fMXWaitFileTimeOut = 30 # s # Flag for using the thin client self.bUseThinClient = True self.edPluginISPyB = None self.listPluginMOSFLM = [] self.defaultMinImageSize = 1000000 self.minImageSize = None def checkParameters(self): """ Checks the mandatory parameters """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.checkParameters") self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.configure") self.fMXWaitFileTimeOut = float(self.config.get("MXWaitFileTimeOut", self.fMXWaitFileTimeOut)) self.minImageSize = self.config.get("minImageSize") if self.minImageSize is None: self.minImageSize = self.defaultMinImageSize def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value self.screen("Batch size: {0}".format(batchSize)) # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators() listPluginDistl = [] listPluginDozor = [] listBatch = [] indexBatch = 0 listH5FilePath = [] ispybDataCollection = None for xsDataImage in listXSDataImage: strPathToImage = xsDataImage.path.value # If Eiger, just wait for the h5 file if "id30a3" in strPathToImage: h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage, batchSize) # print(h5FilePath) # print(hdf5ImageNumber) if not h5DataFilePath in listH5FilePath: self.screen("ID30a3 Eiger data, waiting for master and data files...") listH5FilePath.append(h5DataFilePath) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Waiting for file {0}".format(h5DataFilePath)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() # hdf5FilePath = strPathToImage.replace(".cbf", ".h5") ispybDataCollection = None time.sleep(1) indexLoop = 1 continueLoop = True while continueLoop: xsDataInputControlH5ToCBF = XSDataInputControlH5ToCBF() xsDataInputControlH5ToCBF.hdf5File = XSDataFile(XSDataString(strPathToImage)) imageNumber = EDUtilsImage.getImageNumber(strPathToImage) xsDataInputControlH5ToCBF.imageNumber = XSDataInteger(imageNumber) xsDataInputControlH5ToCBF.hdf5ImageNumber = XSDataInteger(hdf5ImageNumber) xsDataInputControlH5ToCBF.ispybDataCollection = ispybDataCollection edPluginControlH5ToCBF = self.loadPlugin(self.strPluginControlH5ToCBF, "ControlH5ToCBF_%04d_%02d" % (imageNumber, indexLoop)) edPluginControlH5ToCBF.dataInput = xsDataInputControlH5ToCBF edPluginControlH5ToCBF.executeSynchronous() cbfFile = edPluginControlH5ToCBF.dataOutput.outputCBFFile # print(cbfFile) # print(indexLoop) if cbfFile is not None: strPathToImage = cbfFile.path.value # print(cbfFile.path.value) if os.path.exists(strPathToImage): self.screen("Image has been converted to CBF file: {0}".format(strPathToImage)) continueLoop = False # print(continueLoop) if continueLoop: self.screen("Still waiting for converting to CBF file: {0}".format(strPathToImage)) indexLoop += 1 time.sleep(5) if indexLoop > 10: continueLoop = False ispybDataCollection = edPluginControlH5ToCBF.dataOutput.ispybDataCollection elif not os.path.exists(strPathToImage): self.screen("Waiting for file {0}".format(strPathToImage)) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(strPathToImage): strError = "Time-out while waiting for image %s" % strPathToImage self.error(strError) self.addErrorMessage(strError) self.setFailure() else: # Check if we should run distl.signalStrength xsDataImageNew = XSDataImage(XSDataString(strPathToImage)) edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName) self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength() xsDataInputDistlSignalStrength.setReferenceImage(xsDataImageNew) edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append((xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator)) listBatch.append(xsDataImageNew.copy()) if len(listBatch) == batchSize: edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append((edPluginControlDozor, listBatch)) listBatch = [] if len(listBatch) > 0: # Process the remaining images... edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append([edPluginControlDozor, listBatch]) listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators() for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile if imageDozor.spotFile is not None: if os.path.exists(imageDozor.spotFile.path.value): numpyArray = numpy.loadtxt(imageDozor.spotFile.path.value, skiprows=3) xsDataImageQualityIndicators.dozorSpotList = XSDataString(base64.b64encode(numpyArray.tostring())) xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[0])) if len(numpyArray.shape) > 1: xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[1])) xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString( edPluginControlDozor.dataOutput.inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin(self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator]) edPluginMOSFLMIndexing.setDataInput(xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution # print xsDataIndexingResult.marshal() # xsDataResultISPyB = edPluginISPyB.dataOutput # if xsDataResultISPyB is not None: # print xsDataResultISPyB.marshal() def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) if self.edPluginISPyB is not None: # Synchronize ISPyB plugin self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.finallyProcess") self.edPluginISPyB.synchronize() listId = [] for xsDataInteger in self.edPluginISPyB.dataOutput.imageQualityIndicatorsId: listId.append(xsDataInteger.value) self.DEBUG("ISPyB imageQualityIndicatorIds = %r" % listId) self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def generateExecutiveSummary(self, _edPlugin=None): self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.generateExecutiveSummary") self.addErrorWarningMessagesToExecutiveSummary("Image quality indicator plugin execution failure! Error messages: ") self.addExecutiveSummaryLine("Summary of image quality indicators with %s :" % self.strPluginName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary(edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False) def getH5FilePath(self, filePath, batchSize=1): imageNumber = EDUtilsImage.getImageNumber(filePath) prefix = EDUtilsImage.getPrefix(filePath) h5FileNumber = int((imageNumber - 1) / batchSize) * batchSize + 1 h5MasterFileName = "{prefix}_{h5FileNumber}_master.h5".format(prefix=prefix, h5FileNumber=h5FileNumber) h5MasterFilePath = os.path.join(os.path.dirname(filePath), h5MasterFileName) h5DataFileName = "{prefix}_{h5FileNumber}_data_000001.h5".format(prefix=prefix, h5FileNumber=h5FileNumber) h5DataFilePath = os.path.join(os.path.dirname(filePath), h5DataFileName) return h5MasterFilePath, h5DataFilePath, h5FileNumber
class EDPluginControlGridScreeningv1_0(EDPluginControl): """ This plugin is a "light-weight" characterisation to be used for processing images from grid scans. """ def __init__(self): EDPluginControl.__init__(self) self.setXSDataInputClass(XSDataInputGridScreening) self.strControlReadImageHeaderPluginName = "EDPluginControlReadImageHeaderv10" self.edPluginControlReadImageHeader = None self.strControlledIndicatorsPluginName = "EDPluginControlImageQualityIndicatorsv1_1" self.edPluginControlIndicators = None self.strISPyBStoreImageQualityIndicatorsPluginName = "EDPluginISPyBStoreImageQualityIndicatorsv1_3" self.edPluginISPyBStoreImageQualityIndicators = None self.strIndexingMOSFLMPluginName = "EDPluginMOSFLMIndexingv10" self.edPluginMOSFLMIndexing = None self.strPluginControlIntegration = "EDPluginControlIntegrationv10" self.edPluginControlIntegration = None self.strPluginControlStrategy = "EDPluginControlStrategyv1_2" self.edPluginControlStrategy = None self.strPluginExecMtz2Various = "EDPluginExecMtz2Variousv1_0" self.edPluginExecMtz2Various = None self.strImageFile = None self.xsDataIndexingResultMOSFLM = None self.xsDataCrystal = None self.strCharacterisationShortSummary = "" self.strStatusMessage = "" self.xsDataDiffractionPlan = None self.xsDataCollection = None self.xsDataIndexingResult = None self.xsDataStrategyResult = None self.xsDataImageQualityIndicators = None self.bStoreImageQualityIndicatorsInISPyB = False self.bDoOnlyImageQualityIndicators = False self.bDoOnlyIntegrationWithXMLOutput = False self.iImageQualityIndicatorsId = None self.xsDataGridScreeningResultIntegration = None def checkParameters(self): """ Checks the mandatory parameters. """ self.DEBUG("EDPluginControlGridScreeningv1_0.checkParameters") self.checkMandatoryParameters(self.getDataInput(), "Data Input is None") self.checkMandatoryParameters(self.getDataInput().getImageFile(), "imageFile") def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG("EDPluginControlGridScreeningv1_0.preProcess") # Load the plugins self.edPluginControlReadImageHeader = self.loadPlugin(self.strControlReadImageHeaderPluginName, \ "ReadImageHeader") self.edPluginControlIndicators = self.loadPlugin(self.strControlledIndicatorsPluginName, \ "ControlIndicators") self.edPluginMOSFLMIndexing = self.loadPlugin(self.strIndexingMOSFLMPluginName, \ "IndexingMOSFLM") self.edPluginControlIntegration = self.loadPlugin(self.strPluginControlIntegration, \ "Integration") self.edPluginControlStrategy = self.loadPlugin(self.strPluginControlStrategy, \ "Strategy") self.edPluginExecMtz2Various = self.loadPlugin(self.strPluginExecMtz2Various, \ "Mtz2Various") # Input data self.strImageFile = self.getDataInput().getImageFile().getPath( ).getValue() self.xsDataGridScreeningFileNameParameters = self.getFileNameParameters( self.strImageFile) self.xsDataDiffractionPlan = self.getDataInput().getDiffractionPlan() if self.xsDataDiffractionPlan is None: self.xsDataDiffractionPlan = XSDataDiffractionPlan() if self.xsDataDiffractionPlan.getMaxExposureTimePerDataCollection( ) is None: # Default max esposure time: 10000s self.xsDataDiffractionPlan.setMaxExposureTimePerDataCollection( XSDataTime(10000)) self.xsDataDiffractionPlan.setEstimateRadiationDamage( XSDataBoolean(False)) # Image quality indicators if self.getDataInput().getStoreImageQualityIndicatorsInISPyB(): self.bStoreImageQualityIndicatorsInISPyB = self.getDataInput( ).getStoreImageQualityIndicatorsInISPyB().getValue() if self.getDataInput().getDoOnlyImageQualityIndicators(): self.bDoOnlyImageQualityIndicators = self.getDataInput( ).getDoOnlyImageQualityIndicators().getValue() if self.getDataInput().getDoOnlyIntegrationWithXMLOutput(): self.bDoOnlyIntegrationWithXMLOutput = self.getDataInput( ).getDoOnlyIntegrationWithXMLOutput().getValue() if self.bStoreImageQualityIndicatorsInISPyB: self.edPluginISPyBStoreImageQualityIndicators = self.loadPlugin(self.strISPyBStoreImageQualityIndicatorsPluginName, \ "ISPyBStoreImageQualityIndicators") def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlGridScreeningv1_0.process") xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.setImage(image=XSDataFile( path=XSDataString(self.strImageFile))) self.edPluginControlReadImageHeader.setDataInput( xsDataInputReadImageHeader) self.edPluginControlReadImageHeader.connectSUCCESS( self.doSuccessReadImageHeader) self.edPluginControlReadImageHeader.connectFAILURE( self.doFailureReadImageHeader) self.executePluginSynchronous(self.edPluginControlReadImageHeader) def finallyProcess(self, _edObject=None): EDPluginControl.finallyProcess(self) self.DEBUG("EDPluginControlGridScreeningv1_0.finallyProcess") # Synchronise if necessary if self.edPluginISPyBStoreImageQualityIndicators is not None: self.edPluginISPyBStoreImageQualityIndicators.synchronize() # Build up the output object strComment = "" xsDataResultGridScreening = XSDataResultGridScreening() if self.xsDataGridScreeningFileNameParameters is not None: xsDataResultGridScreening.setFileNameParameters( self.xsDataGridScreeningFileNameParameters) if self.xsDataImageQualityIndicators is None: strComment = "No image quality indicators" else: xsDataResultGridScreening.setImageQualityIndicators( self.xsDataImageQualityIndicators) if self.xsDataImageQualityIndicators.getIceRings().getValue() > 1: strComment = "Ice rings detected" if self.xsDataIndexingResult is None: if strComment == "": strComment = "No indexing result" else: strComment += ", no indexing result" else: xsDataSelectedSolution = self.xsDataIndexingResult.getSelectedSolution( ) xsDataResultGridScreening.setMosaicity( xsDataSelectedSolution.getCrystal().getMosaicity()) if self.xsDataStrategyResult is None: if strComment == "": strComment = "No strategy result" else: strComment += ", no strategy result" else: xsDataCollectionPlan = self.xsDataStrategyResult.getCollectionPlan( )[0] xsDataStrategySummary = xsDataCollectionPlan.getStrategySummary( ) xsDataResultGridScreening.setRankingResolution( xsDataStrategySummary.getRankingResolution()) xsDataResultGridScreening.setResultIntegration( self.xsDataGridScreeningResultIntegration) xsDataResultGridScreening.setComment(XSDataString(strComment)) if self.iImageQualityIndicatorsId is not None: xsDataResultGridScreening.setImageQualityIndicatorsId( XSDataInteger(self.iImageQualityIndicatorsId)) #print xsDataResultGridScreening.marshal() self.setDataOutput(xsDataResultGridScreening) def doSuccessReadImageHeader(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doSuccessReadImageHeader") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlGridScreeningv1_0.doSuccessReadImageHeader") xsDataResultReadImageHeader = self.edPluginControlReadImageHeader.getDataOutput( ) if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.getSubWedge() self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) self.xsDataCollection.setDiffractionPlan( self.xsDataDiffractionPlan) if not self.bDoOnlyIntegrationWithXMLOutput: xsDataInputControlImageQualityIndicators = XSDataInputControlImageQualityIndicators( ) xsDataInputControlImageQualityIndicators.addImage( XSDataImage(path=XSDataString(self.strImageFile))) self.edPluginControlIndicators.setDataInput( xsDataInputControlImageQualityIndicators) self.edPluginControlIndicators.connectSUCCESS( self.doSuccessIndicators) self.edPluginControlIndicators.connectFAILURE( self.doFailureIndicators) self.executePluginSynchronous(self.edPluginControlIndicators) else: xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(self.xsDataCollection) from EDHandlerXSDataMOSFLMv10 import EDHandlerXSDataMOSFLMv10 xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing( xsDataIndexingInput) self.edPluginMOSFLMIndexing.connectSUCCESS( self.doSuccessIndexingMOSFLM) self.edPluginMOSFLMIndexing.connectFAILURE( self.doFailureIndexingMOSFLM) self.edPluginMOSFLMIndexing.setDataInput( xsDataMOSFLMIndexingInput) self.edPluginMOSFLMIndexing.executeSynchronous() def doFailureReadImageHeader(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doFailureReadImageHeader") self.retrieveFailureMessages( _edPlugin, "EDPluginControlGridScreeningv1_0.doSuccessReadImageHeader") def doSuccessIndicators(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doSuccessIndicators") #self.retrieveSuccessMessages(_edPlugin, "EDPluginControlGridScreeningv1_0.doSuccessIndexingIndicators") if self.edPluginControlIndicators.hasDataOutput(): self.xsDataImageQualityIndicators = self.edPluginControlIndicators.getDataOutput( ).getImageQualityIndicators()[0] # Store results in ISPyB if requested if self.bStoreImageQualityIndicatorsInISPyB: xsDataInputStoreImageQualityIndicators = XSDataInputStoreImageQualityIndicators( ) xsDataISPyBImageQualityIndicators = XSDataISPyBImageQualityIndicators.parseString( self.xsDataImageQualityIndicators.marshal()) xsDataInputStoreImageQualityIndicators.setImageQualityIndicators( xsDataISPyBImageQualityIndicators) self.edPluginISPyBStoreImageQualityIndicators.setDataInput( xsDataInputStoreImageQualityIndicators) self.edPluginISPyBStoreImageQualityIndicators.connectSUCCESS( self.doSuccessISPyBStoreImageQualityIndicators) self.edPluginISPyBStoreImageQualityIndicators.connectFAILURE( self.doFailureISPyBStoreImageQualityIndicators) self.edPluginISPyBStoreImageQualityIndicators.execute() # Continue only if requested if not self.bDoOnlyImageQualityIndicators: xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(self.xsDataCollection) from EDHandlerXSDataMOSFLMv10 import EDHandlerXSDataMOSFLMv10 xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing( xsDataIndexingInput) self.edPluginMOSFLMIndexing.connectSUCCESS( self.doSuccessIndexingMOSFLM) self.edPluginMOSFLMIndexing.connectFAILURE( self.doFailureIndexingMOSFLM) self.edPluginMOSFLMIndexing.setDataInput( xsDataMOSFLMIndexingInput) self.edPluginMOSFLMIndexing.executeSynchronous() def doFailureIndicators(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doFailureIndicators") strErrorMessage = "Execution of Indexing and Indicators plugin failed. Execution of characterisation aborted." self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) def doSuccessISPyBStoreImageQualityIndicators(self, _edPlugin=None): self.DEBUG( "EDPluginControlGridScreeningv1_0.doSuccessISPyBStoreImageQualityIndicators" ) self.retrieveSuccessMessages( _edPlugin, "EDPluginControlGridScreeningv1_0.doSuccessISPyBStoreImageQualityIndicators" ) self.iImageQualityIndicatorsId = self.edPluginISPyBStoreImageQualityIndicators.getDataOutput( ).getImageQualityIndicatorsId().getValue() if self.iImageQualityIndicatorsId is None: self.WARNING( "Image quality indicators result not stored in ISPyB!") def doFailureISPyBStoreImageQualityIndicators(self, _edPlugin=None): self.DEBUG( "EDPluginControlGridScreeningv1_0.doFailureISPyBStoreImageQualityIndicators" ) strErrorMessage = "Execution of store ISPyB image quality indicators plugin failed." self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) def doSuccessIndexingMOSFLM(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doSuccessIndexingMOSFLM") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlGridScreeningv1_0.doIntegrationToStrategyTransition" ) xsDataMOSFLMIndexingOutput = self.edPluginMOSFLMIndexing.getDataOutput( ) xsDataExperimentalConditionRefined = None if self.hasDataInput("refinedExperimentalCondition"): xsDataExperimentalConditionRefined = self.getDataInput( "refinedExperimentalCondition")[0] else: # Use the experimental condition from the xsDataCollection xsDataExperimentalConditionRefined = self.xsDataCollection.getSubWedge( )[0].getExperimentalCondition() from EDHandlerXSDataMOSFLMv10 import EDHandlerXSDataMOSFLMv10 self.xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult( xsDataMOSFLMIndexingOutput, xsDataExperimentalConditionRefined) xsDataIntegrationInput = XSDataIntegrationInput() xsDataIntegrationInput.setDataCollection(self.xsDataCollection) xsDataIntegrationInput.setExperimentalConditionRefined( self.xsDataIndexingResult.getSelectedSolution( ).getExperimentalConditionRefined()) xsDataIntegrationInput.setSelectedIndexingSolution( self.xsDataIndexingResult.getSelectedSolution()) self.edPluginControlIntegration.connectSUCCESS( self.doSuccessIntegration) self.edPluginControlIntegration.connectFAILURE( self.doFailureIntegration) self.edPluginControlIntegration.setDataInput(xsDataIntegrationInput) self.executePluginSynchronous(self.edPluginControlIntegration) def doFailureIndexingMOSFLM(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doFailureActionIndexing") self.retrieveFailureMessages( _edPlugin, "EDPluginControlCharacterisationv1_1.doFailureActionIndexing") def doSuccessIntegration(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doSuccessIntegration") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlGridScreeningv1_0.doSuccessIntegration") self.addStatusMessage("Integration successful.") self.xsDataIntegrationOutput = self.edPluginControlIntegration.getDataOutput( ) # Integration short summary if self.edPluginControlIntegration.hasDataOutput( "integrationShortSummary"): self.strCharacterisationShortSummary += self.edPluginControlIntegration.getDataOutput( "integrationShortSummary")[0].getValue() #self.DEBUG( self.xsDataExperimentCharacterisation.marshal() ) if self.bDoOnlyIntegrationWithXMLOutput: # Run mtz2various xsDataInputMtz2Various = XSDataInputMtz2Various() xsDataInputMtz2Various.setMtzfile( self.edPluginControlIntegration.getDataOutput( ).getIntegrationSubWedgeResult()[0].getGeneratedMTZFile()) xsDataInputMtz2Various.addLabin(XSDataString("I=I")) xsDataInputMtz2Various.addLabin(XSDataString("SIGI=SIGI")) xsDataInputMtz2Various.setOutput( XSDataString("USER '(3I4,2F10.1)'")) self.edPluginExecMtz2Various.setDataInput(xsDataInputMtz2Various) self.edPluginExecMtz2Various.executeSynchronous() strHklFilePath = self.edPluginExecMtz2Various.getDataOutput( ).getHklfile().getPath().getValue() strIntegration = EDUtilsFile.readFile(strHklFilePath) # Output the result in XML format self.xsDataGridScreeningResultIntegration = XSDataGridScreeningResultIntegration( ) self.xsDataGridScreeningResultIntegration.setFileName( os.path.basename(self.strImageFile)) self.xsDataGridScreeningResultIntegration.setFileDirectory( os.path.dirname(self.strImageFile)) self.xsDataGridScreeningResultIntegration.setIntegratedData( strIntegration) else: # We continue with the strategy calculation xsDataInputStrategy = XSDataInputStrategy() xsDataSolutionSelected = self.xsDataIndexingResult.getSelectedSolution( ) xsDataInputStrategy.setCrystalRefined( xsDataSolutionSelected.getCrystal()) xsDataInputStrategy.setSample(self.xsDataCollection.getSample()) xsDataIntegrationSubWedgeResultList = self.xsDataIntegrationOutput.getIntegrationSubWedgeResult( ) xsDataInputStrategy.setBestFileContentDat( xsDataIntegrationSubWedgeResultList[0].getBestfileDat()) xsDataInputStrategy.setBestFileContentPar( xsDataIntegrationSubWedgeResultList[0].getBestfilePar()) xsDataInputStrategy.setExperimentalCondition( xsDataIntegrationSubWedgeResultList[0]. getExperimentalConditionRefined()) for xsDataIntegrationSubWedgeResult in xsDataIntegrationSubWedgeResultList: xsDataInputStrategy.addBestFileContentHKL( xsDataIntegrationSubWedgeResult.getBestfileHKL()) xsDataInputStrategy.setDiffractionPlan(self.xsDataDiffractionPlan) self.edPluginControlStrategy.connectSUCCESS(self.doSuccessStrategy) self.edPluginControlStrategy.connectFAILURE(self.doFailureStrategy) self.edPluginControlStrategy.setDataInput(xsDataInputStrategy) self.executePluginSynchronous(self.edPluginControlStrategy) def doFailureIntegration(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doFailureIntegration") strErrorMessage = "Execution of integration plugin failed." self.addStatusMessage("Integration FAILURE.") self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) def doSuccessStrategy(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doSuccessStrategy") self.retrieveSuccessMessages( self.edPluginControlStrategy, "EDPluginControlGridScreeningv1_0.doSuccessStrategy") self.xsDataStrategyResult = self.edPluginControlStrategy.getDataOutput( ) def doFailureStrategy(self, _edPlugin=None): self.DEBUG("EDPluginControlGridScreeningv1_0.doFailureStrategy") self.retrieveFailureMessages( self.edPluginControlStrategy, "EDPluginControlGridScreeningv1_0.doFailureStrategy") strErrorMessage = "Execution of strategy plugin failed." self.ERROR(strErrorMessage) self.addErrorMessage(strErrorMessage) # def generateExecutiveSummary(self, _edPlugin): # """ # Generates a summary of the execution of the plugin. # """ # self.DEBUG("EDPluginControlGridScreeningv1_0.generateExecutiveSummary") # self.addExecutiveSummaryLine("Summary of characterisation:") # xsDataInputStrategy = self.getDataInput() # xsDataCollection = xsDataInputStrategy.getDataCollection() # xsDataDiffractionPlan = xsDataCollection.getDiffractionPlan() # self.addExecutiveSummaryLine("Diffraction plan:") # if (xsDataDiffractionPlan.getComplexity() is not None): # self.addExecutiveSummaryLine("BEST complexity : %s" % xsDataDiffractionPlan.getComplexity().getValue()) # if (xsDataDiffractionPlan.getAimedCompleteness() is not None): # self.addExecutiveSummaryLine("Aimed completeness : %6.1f [%%]" % (100.0 * xsDataDiffractionPlan.getAimedCompleteness().getValue())) # if (xsDataDiffractionPlan.getRequiredCompleteness() is not None): # self.addExecutiveSummaryLine("Required completeness : %6.1f [%%]" % (100.0 * xsDataDiffractionPlan.getRequiredCompleteness().getValue())) # if (xsDataDiffractionPlan.getAimedIOverSigmaAtHighestResolution() is not None): # self.addExecutiveSummaryLine("Aimed I/sigma at highest resolution : %6.1f" % xsDataDiffractionPlan.getAimedIOverSigmaAtHighestResolution().getValue()) # if (xsDataDiffractionPlan.getAimedResolution() is not None): # self.addExecutiveSummaryLine("Aimed resolution : %6.1f [A]" % xsDataDiffractionPlan.getAimedResolution().getValue()) # if (xsDataDiffractionPlan.getRequiredResolution() is not None): # self.addExecutiveSummaryLine("Required resolution : %6.1f [A]" % xsDataDiffractionPlan.getRequiredResolution().getValue()) # if (xsDataDiffractionPlan.getAimedMultiplicity() is not None): # self.addExecutiveSummaryLine("Aimed multiplicity : %6.1f" % xsDataDiffractionPlan.getAimedMultiplicity().getValue()) # if (xsDataDiffractionPlan.getRequiredMultiplicity() is not None): # self.addExecutiveSummaryLine("Required multiplicity : %6.1f" % xsDataDiffractionPlan.getRequiredMultiplicity().getValue()) # if (xsDataDiffractionPlan.getForcedSpaceGroup() is not None): # self.addExecutiveSummaryLine("Forced space group : %6s" % xsDataDiffractionPlan.getForcedSpaceGroup().getValue()) # if (xsDataDiffractionPlan.getMaxExposureTimePerDataCollection() is not None): # self.addExecutiveSummaryLine("Max exposure time per data collection : %6.1f [s]" % xsDataDiffractionPlan.getMaxExposureTimePerDataCollection().getValue()) # self.addExecutiveSummarySeparator() # if self.edPluginControlIndexingIndicators is not None: # self.appendExecutiveSummary(self.edPluginControlIndexingIndicators, "") # if self.edPluginControlIndexingLabelit is not None: # self.appendExecutiveSummary(self.edPluginControlIndexingLabelit, "") # if self.edPluginControlIntegration is not None: # self.appendExecutiveSummary(self.edPluginControlIntegration, "") # if self.edPluginControlStrategy is not None: # self.appendExecutiveSummary(self.edPluginControlStrategy, "") # self.addExecutiveSummarySeparator() # if self.strCharacterisationShortSummary is not None: # self.addExecutiveSummaryLine("Characterisation short summary:") # self.addExecutiveSummaryLine("") # if self.strStatusMessage != None: # for strLine in self.strStatusMessage.split(". "): # if strLine.endswith("."): # self.addExecutiveSummaryLine(strLine) # else: # self.addExecutiveSummaryLine(strLine + ".") # self.addExecutiveSummaryLine("") # for strLine in self.strCharacterisationShortSummary.split("\n"): # if strLine != "\n": # self.addExecutiveSummaryLine(strLine) # self.addErrorWarningMessagesToExecutiveSummary("Characterisation error and warning messages: ") # self.addExecutiveSummarySeparator() def addStatusMessage(self, _strStatusMessage): if self.strStatusMessage != "": self.strStatusMessage += " " self.strStatusMessage += _strStatusMessage def getFileNameParameters(self, _strFileName): """Method for extracting the rotation start angle, the two motor positions and the grid scan image no from the file name""" # Typical file name: mesh_0_21.676_-0.051_22_001.mccd listParts = os.path.basename(_strFileName).split("_") xsDataGridScreeningFileNameParameters = XSDataGridScreeningFileNameParameters( ) try: strScanId1 = listParts[1] xsDataGridScreeningFileNameParameters.setScanId1( XSDataString(strScanId1)) strMotorPosition1 = listParts[2] xsDataGridScreeningFileNameParameters.setMotorPosition1( XSDataString(strMotorPosition1)) strMotorPosition2 = listParts[3] xsDataGridScreeningFileNameParameters.setMotorPosition2( XSDataString(strMotorPosition2)) strScanId2 = listParts[4] xsDataGridScreeningFileNameParameters.setScanId2( XSDataString(strScanId2)) except: xsDataGridScreeningFileNameParameters = None return xsDataGridScreeningFileNameParameters
class EDPluginControlImageQualityIndicatorsv1_4(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__ (self): EDPluginControl.__init__(self) self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1" self.strPluginName = "EDPluginDistlSignalStrengthv1_1" self.strPluginNameThinClient = "EDPluginDistlSignalStrengthThinClientv1_1" self.strPluginNameControlDozor = "EDPluginControlDozorv1_0" self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_4" self.strIndexingMOSFLMPluginName = "EDPluginMOSFLMIndexingv10" self.edPluginMOSFLMIndexing = None self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.edPluginReadImageHeader = None self.edPluginControlDozor = None self.strPluginControlH5ToCBF = "EDPluginControlH5ToCBFv1_1" self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = [] self.listPluginControlDozor = [] self.xsDataResultControlImageQualityIndicators = None self.edPluginMXWaitFile = None # Default time out for wait file self.fMXWaitFileTimeOut = 120 # s # Flag for using the thin client - disabled as of 2016/07/20 self.bUseThinClient = False self.edPluginISPyB = None self.listPluginMOSFLM = [] self.defaultMinImageSize = 1000000 self.minImageSize = None self.strEDPluginControlReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.hasOverlap = False def checkParameters(self): """ Checks the mandatory parameters """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.checkParameters") # self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.configure") self.fMXWaitFileTimeOut = float(self.config.get("MXWaitFileTimeOut", self.fMXWaitFileTimeOut)) self.minImageSize = self.config.get("minImageSize") if self.minImageSize is None: self.minImageSize = self.defaultMinImageSize def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value self.screen("Image quality indicators batch size: {0}".format(batchSize)) # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Check if fast mesh (for HDF5) isFastMesh = False if self.dataInput.fastMesh: isFastMesh = self.dataInput.fastMesh.value # Loop through all the incoming reference images if len(self.dataInput.image) == 0: directory = self.dataInput.directory.path.value template = self.dataInput.template.value startNo = self.dataInput.startNo.value endNo = self.dataInput.endNo.value listXSDataImage = [] for index in range(startNo, endNo + 1): imageName = template.replace("####", "{0:04d}".format(index)) imagePath = os.path.join(directory, imageName) xsDataImage = XSDataImage(path=XSDataString(imagePath), number=XSDataInteger(index)) listXSDataImage.append(xsDataImage) else: listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators() listPluginDistl = [] listPluginDozor = [] listOfImagesInBatch = [] listOfAllBatches = [] indexBatch = 0 listH5FilePath = [] # Process data in batches for xsDataImage in listXSDataImage: listOfImagesInBatch.append(xsDataImage.copy()) if len(listOfImagesInBatch) == batchSize: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] if len(listOfImagesInBatch) > 0: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] # Loop over batches for listOfImagesInBatch in listOfAllBatches: # First wait for images for image in listOfImagesInBatch: strPathToImage = image.path.value # If Eiger, just wait for the h5 file if strPathToImage.endswith(".h5"): h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage, batchSize=batchSize, isFastMesh=isFastMesh) # print(h5FilePath) # print(hdf5ImageNumber) if not h5DataFilePath in listH5FilePath: self.screen("ID30a3 Eiger data, waiting for master and data files...") listH5FilePath.append(h5DataFilePath) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Waiting for file {0}".format(h5DataFilePath)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() # hdf5FilePath = strPathToImage.replace(".cbf", ".h5") time.sleep(1) if not os.path.exists(h5DataFilePath): strError = "Time-out while waiting for image %s" % h5DataFilePath self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if not os.path.exists(strPathToImage): # self.screen("Waiting for file {0}".format(strPathToImage)) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Wait file timeOut set to %.0f s" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(strPathToImage): strError = "Time-out while waiting for image %s" % strPathToImage self.error(strError) self.addErrorMessage(strError) self.setFailure() if not self.isFailure(): strPathToFirstImage = listOfImagesInBatch[0].path.value if strPathToImage.endswith(".h5"): indexLoop = 1 continueLoop = True while continueLoop: directory = os.path.dirname(strPathToFirstImage) firstImage = EDUtilsImage.getImageNumber(listOfImagesInBatch[0].path.value) lastImage = EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value) xsDataInputH5ToCBF = XSDataInputH5ToCBF() xsDataInputH5ToCBF.hdf5File = XSDataFile(listOfImagesInBatch[0].path) xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1) xsDataInputH5ToCBF.startImageNumber = XSDataInteger(firstImage) xsDataInputH5ToCBF.endImageNumber = XSDataInteger(lastImage) xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile(XSDataString(directory)) edPluginH5ToCBF = self.loadPlugin("EDPluginH5ToCBFv1_1") edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF edPluginH5ToCBF.execute() edPluginH5ToCBF.synchronize() outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate if outputCBFFileTemplate is not None: lastCbfFile = outputCBFFileTemplate.path.value.replace("######", "{0:06d}".format(EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value))) strPathToImage = os.path.join(directory, lastCbfFile) # print(cbfFile.path.value) if os.path.exists(strPathToImage): # Rename all images for image in listOfImagesInBatch: image.path.value = image.path.value.replace(".h5", ".cbf") imageNumber = EDUtilsImage.getImageNumber(image.path.value) oldPath = os.path.join(directory, outputCBFFileTemplate.path.value.replace("######", "{0:06d}".format(imageNumber))) newPath = os.path.join(directory, outputCBFFileTemplate.path.value.replace("######", "{0:04d}".format(imageNumber))) os.rename(oldPath, newPath) lastCbfFile = outputCBFFileTemplate.path.value.replace("######", "{0:04d}".format(EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value))) strPathToImage = os.path.join(directory, lastCbfFile) self.screen("Image has been converted to CBF file: {0}".format(strPathToImage)) continueLoop = False # print(continueLoop) if continueLoop: self.screen("Still waiting for converting to CBF file: {0}".format(strPathToImage)) indexLoop += 1 time.sleep(5) if indexLoop > 10: continueLoop = False for image in listOfImagesInBatch: strPathToImage = image.path.value # Check if we should run distl.signalStrength xsDataImageNew = XSDataImage(XSDataString(strPathToImage)) xsDataImageNew.number = XSDataInteger(EDUtilsImage.getImageNumber(image.path.value)) edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName) self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength() xsDataInputDistlSignalStrength.setReferenceImage(xsDataImageNew) edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append((xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator)) edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor, "ControlDozor_{0}".format(os.path.splitext(os.path.basename(strPathToFirstImage))[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listOfImagesInBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(len(listOfImagesInBatch)) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append((edPluginControlDozor, list(listOfImagesInBatch))) if not self.isFailure(): listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators() for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput is not None: if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() # Check that we got at least one result if len(edPluginControlDozor.dataOutput.imageDozor) == 0: # Run the dozor plugin again, this time synchronously firstImage = os.path.basename(listBatch[0].path.value) lastImage = os.path.basename(listBatch[-1].path.value) self.screen("No dozor results! Re-executing Dozor for images {0} to {1}".format(firstImage, lastImage)) time.sleep(5) edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor, "ControlDozor_reexecution_{0}".format(os.path.splitext(firstImage)[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.executeSynchronous() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile if imageDozor.spotFile is not None: if os.path.exists(imageDozor.spotFile.path.value): numpyArray = numpy.loadtxt(imageDozor.spotFile.path.value, skiprows=3) xsDataImageQualityIndicators.dozorSpotList = XSDataString(base64.b64encode(numpyArray.tostring())) xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[0])) if len(numpyArray.shape) > 1: xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[1])) xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString( edPluginControlDozor.dataOutput.inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.dozor_score.value > 1: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin(self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator]) edPluginMOSFLMIndexing.setDataInput(xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) if self.edPluginISPyB is not None: # Synchronize ISPyB plugin self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.finallyProcess") self.edPluginISPyB.synchronize() listId = [] for xsDataInteger in self.edPluginISPyB.dataOutput.imageQualityIndicatorsId: listId.append(xsDataInteger.value) self.DEBUG("ISPyB imageQualityIndicatorIds = %r" % listId) self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def generateExecutiveSummary(self, _edPlugin=None): self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.generateExecutiveSummary") self.addErrorWarningMessagesToExecutiveSummary("Image quality indicator plugin execution failure! Error messages: ") self.addExecutiveSummaryLine("Summary of image quality indicators with %s :" % self.strPluginName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary(edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False) def getH5FilePath(self, filePath, batchSize=1, isFastMesh=False): imageNumber = EDUtilsImage.getImageNumber(filePath) prefix = EDUtilsImage.getPrefix(filePath) if isFastMesh: h5ImageNumber = int((imageNumber - 1) / 100) + 1 h5FileNumber = 1 else: h5ImageNumber = 1 h5FileNumber = int((imageNumber - 1) / batchSize) * batchSize + 1 h5MasterFileName = "{prefix}_{h5FileNumber}_master.h5".format(prefix=prefix, h5FileNumber=h5FileNumber) h5MasterFilePath = os.path.join(os.path.dirname(filePath), h5MasterFileName) h5DataFileName = "{prefix}_{h5FileNumber}_data_{h5ImageNumber:06d}.h5".format(prefix=prefix, h5FileNumber=h5FileNumber, h5ImageNumber=h5ImageNumber) h5DataFilePath = os.path.join(os.path.dirname(filePath), h5DataFileName) return h5MasterFilePath, h5DataFilePath, h5FileNumber
class EDPluginControlImageQualityIndicatorsv1_4(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__(self): EDPluginControl.__init__(self) self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1" self.strPluginName = "EDPluginDistlSignalStrengthv1_1" self.strPluginNameThinClient = "EDPluginDistlSignalStrengthThinClientv1_1" self.strPluginNameControlDozor = "EDPluginControlDozorv1_0" self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_4" self.strIndexingMOSFLMPluginName = "EDPluginMOSFLMIndexingv10" self.edPluginMOSFLMIndexing = None self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.edPluginReadImageHeader = None self.edPluginControlDozor = None self.strPluginControlH5ToCBF = "EDPluginControlH5ToCBFv1_1" self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = [] self.listPluginControlDozor = [] self.xsDataResultControlImageQualityIndicators = None self.edPluginMXWaitFile = None # Default time out for wait file self.fMXWaitFileTimeOut = 120 # s # Flag for using the thin client - disabled as of 2016/07/20 self.bUseThinClient = False self.edPluginISPyB = None self.listPluginMOSFLM = [] self.defaultMinImageSize = 1000000 self.minImageSize = None self.strEDPluginControlReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.hasOverlap = False def checkParameters(self): """ Checks the mandatory parameters """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.checkParameters") # self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.configure") self.fMXWaitFileTimeOut = float( self.config.get("MXWaitFileTimeOut", self.fMXWaitFileTimeOut)) self.minImageSize = self.config.get("minImageSize") if self.minImageSize is None: self.minImageSize = self.defaultMinImageSize def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value self.screen( "Image quality indicators batch size: {0}".format(batchSize)) # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Check if fast mesh (for HDF5) isFastMesh = False if self.dataInput.fastMesh: isFastMesh = self.dataInput.fastMesh.value # Loop through all the incoming reference images if len(self.dataInput.image) == 0: directory = self.dataInput.directory.path.value template = self.dataInput.template.value startNo = self.dataInput.startNo.value endNo = self.dataInput.endNo.value listXSDataImage = [] for index in range(startNo, endNo + 1): imageName = template.replace("####", "{0:04d}".format(index)) imagePath = os.path.join(directory, imageName) xsDataImage = XSDataImage(path=XSDataString(imagePath), number=XSDataInteger(index)) listXSDataImage.append(xsDataImage) else: listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPluginDistl = [] listPluginDozor = [] listOfImagesInBatch = [] listOfAllBatches = [] indexBatch = 0 listH5FilePath = [] # Process data in batches for xsDataImage in listXSDataImage: listOfImagesInBatch.append(xsDataImage.copy()) if len(listOfImagesInBatch) == batchSize: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] if len(listOfImagesInBatch) > 0: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] # Loop over batches for listOfImagesInBatch in listOfAllBatches: # First wait for images for image in listOfImagesInBatch: strPathToImage = image.path.value # If Eiger, just wait for the h5 file if strPathToImage.endswith(".h5"): h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath( strPathToImage, batchSize=batchSize, isFastMesh=isFastMesh) # print(h5FilePath) # print(hdf5ImageNumber) if not h5DataFilePath in listH5FilePath: self.screen( "ID30a3 Eiger data, waiting for master and data files..." ) listH5FilePath.append(h5DataFilePath) self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile( XSDataString(h5DataFilePath)) xsDataInputMXWaitFile.setSize( XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.screen( "Waiting for file {0}".format(h5DataFilePath)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput( xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() # hdf5FilePath = strPathToImage.replace(".cbf", ".h5") time.sleep(1) if not os.path.exists(h5DataFilePath): strError = "Time-out while waiting for image %s" % h5DataFilePath self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if not os.path.exists(strPathToImage): # self.screen("Waiting for file {0}".format(strPathToImage)) self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile( XSDataString(strPathToImage)) xsDataInputMXWaitFile.setSize( XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Wait file timeOut set to %.0f s" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput( xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(strPathToImage): strError = "Time-out while waiting for image %s" % strPathToImage self.error(strError) self.addErrorMessage(strError) self.setFailure() if not self.isFailure(): strPathToFirstImage = listOfImagesInBatch[0].path.value if strPathToImage.endswith(".h5"): indexLoop = 1 continueLoop = True while continueLoop: directory = os.path.dirname(strPathToFirstImage) firstImage = EDUtilsImage.getImageNumber( listOfImagesInBatch[0].path.value) lastImage = EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value) xsDataInputH5ToCBF = XSDataInputH5ToCBF() xsDataInputH5ToCBF.hdf5File = XSDataFile( listOfImagesInBatch[0].path) xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1) xsDataInputH5ToCBF.startImageNumber = XSDataInteger( firstImage) xsDataInputH5ToCBF.endImageNumber = XSDataInteger( lastImage) xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile( XSDataString(directory)) edPluginH5ToCBF = self.loadPlugin( "EDPluginH5ToCBFv1_1") edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF edPluginH5ToCBF.execute() edPluginH5ToCBF.synchronize() outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate if outputCBFFileTemplate is not None: lastCbfFile = outputCBFFileTemplate.path.value.replace( "######", "{0:06d}".format( EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value))) strPathToImage = os.path.join( directory, lastCbfFile) # print(cbfFile.path.value) if os.path.exists(strPathToImage): # Rename all images for image in listOfImagesInBatch: image.path.value = image.path.value.replace( ".h5", ".cbf") imageNumber = EDUtilsImage.getImageNumber( image.path.value) oldPath = os.path.join( directory, outputCBFFileTemplate.path.value. replace("######", "{0:06d}".format(imageNumber))) newPath = os.path.join( directory, outputCBFFileTemplate.path.value. replace("######", "{0:04d}".format(imageNumber))) os.rename(oldPath, newPath) lastCbfFile = outputCBFFileTemplate.path.value.replace( "######", "{0:04d}".format( EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value) )) strPathToImage = os.path.join( directory, lastCbfFile) self.screen( "Image has been converted to CBF file: {0}" .format(strPathToImage)) continueLoop = False # print(continueLoop) if continueLoop: self.screen( "Still waiting for converting to CBF file: {0}" .format(strPathToImage)) indexLoop += 1 time.sleep(5) if indexLoop > 10: continueLoop = False for image in listOfImagesInBatch: strPathToImage = image.path.value # Check if we should run distl.signalStrength xsDataImageNew = XSDataImage(XSDataString(strPathToImage)) xsDataImageNew.number = XSDataInteger( EDUtilsImage.getImageNumber(image.path.value)) edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage( xsDataImageNew) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append( (xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator)) edPluginControlDozor = self.loadPlugin( self.strPluginNameControlDozor, "ControlDozor_{0}".format( os.path.splitext( os.path.basename(strPathToFirstImage))[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listOfImagesInBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger( len(listOfImagesInBatch)) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append( (edPluginControlDozor, list(listOfImagesInBatch))) if not self.isFailure(): listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput is not None: if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() # Check that we got at least one result if len(edPluginControlDozor.dataOutput.imageDozor) == 0: # Run the dozor plugin again, this time synchronously firstImage = os.path.basename(listBatch[0].path.value) lastImage = os.path.basename(listBatch[-1].path.value) self.screen( "No dozor results! Re-executing Dozor for images {0} to {1}" .format(firstImage, lastImage)) time.sleep(5) edPluginControlDozor = self.loadPlugin( self.strPluginNameControlDozor, "ControlDozor_reexecution_{0}".format( os.path.splitext(firstImage)[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile( image.path)) xsDataInputControlDozor.batchSize = XSDataInteger( batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.executeSynchronous() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile if imageDozor.spotFile is not None: if os.path.exists( imageDozor.spotFile.path.value): numpyArray = numpy.loadtxt( imageDozor.spotFile.path.value, skiprows=3) xsDataImageQualityIndicators.dozorSpotList = XSDataString( base64.b64encode( numpyArray.tostring())) xsDataImageQualityIndicators.addDozorSpotListShape( XSDataInteger(numpyArray.shape[0])) if len(numpyArray.shape) > 1: xsDataImageQualityIndicators.addDozorSpotListShape( XSDataInteger(numpyArray.shape[1])) xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput( ).parseString( edPluginControlDozor.dataOutput. inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[ -5:]: if xsDataResultControlImageQualityIndicator.dozor_score.value > 1: xsDataInputReadImageHeader = XSDataInputReadImageHeader( ) xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path ) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection( self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing( xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin( self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([ edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator ]) edPluginMOSFLMIndexing.setDataInput( xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult( xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) if self.edPluginISPyB is not None: # Synchronize ISPyB plugin self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_4.finallyProcess") self.edPluginISPyB.synchronize() listId = [] for xsDataInteger in self.edPluginISPyB.dataOutput.imageQualityIndicatorsId: listId.append(xsDataInteger.value) self.DEBUG("ISPyB imageQualityIndicatorIds = %r" % listId) self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def generateExecutiveSummary(self, _edPlugin=None): self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_4.generateExecutiveSummary" ) self.addErrorWarningMessagesToExecutiveSummary( "Image quality indicator plugin execution failure! Error messages: " ) self.addExecutiveSummaryLine( "Summary of image quality indicators with %s :" % self.strPluginName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary( edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False) def getH5FilePath(self, filePath, batchSize=1, isFastMesh=False): imageNumber = EDUtilsImage.getImageNumber(filePath) prefix = EDUtilsImage.getPrefix(filePath) if isFastMesh: h5ImageNumber = int((imageNumber - 1) / 100) + 1 h5FileNumber = 1 else: h5ImageNumber = 1 h5FileNumber = int((imageNumber - 1) / batchSize) * batchSize + 1 h5MasterFileName = "{prefix}_{h5FileNumber}_master.h5".format( prefix=prefix, h5FileNumber=h5FileNumber) h5MasterFilePath = os.path.join(os.path.dirname(filePath), h5MasterFileName) h5DataFileName = "{prefix}_{h5FileNumber}_data_{h5ImageNumber:06d}.h5".format( prefix=prefix, h5FileNumber=h5FileNumber, h5ImageNumber=h5ImageNumber) h5DataFilePath = os.path.join(os.path.dirname(filePath), h5DataFileName) return h5MasterFilePath, h5DataFilePath, h5FileNumber
def getXSDataResultStrategy(self, _xsDataResultBest, _xsDataExperimentalCondition, _xsDataSample): xsDataResultStrategy = XSDataResultStrategy() listXSDataBestCollectionPlan = _xsDataResultBest.getCollectionPlan() for xsDataBestCollectionPlan in listXSDataBestCollectionPlan: xsDataCollectionPlan = XSDataCollectionPlan() xsDataCollectionStrategy = XSDataCollection() xsDataBestStrategySummary = xsDataBestCollectionPlan.getStrategySummary() xsDataDoubleTransmission = xsDataBestStrategySummary.getTransmission() for xsDataBestCollectionRun in xsDataBestCollectionPlan.getCollectionRun(): xsDataSubWedge = XSDataSubWedge() strXmlStringDataExperimentalCondition = _xsDataExperimentalCondition.marshal() xsDataExperimentalCondition = XSDataExperimentalCondition.parseString( strXmlStringDataExperimentalCondition ) xsDataExperimentalCondition.getBeam().setExposureTime(xsDataBestCollectionRun.getExposureTime()) if xsDataBestCollectionRun.getTransmission() is None: xsDataExperimentalCondition.getBeam().setTransmission(xsDataDoubleTransmission) else: xsDataExperimentalCondition.getBeam().setTransmission(xsDataBestCollectionRun.getTransmission()) xsDataExperimentalCondition.getDetector().setDistance(xsDataBestStrategySummary.getDistance()) xsDataExperimentalCondition.getGoniostat().setRotationAxisStart(xsDataBestCollectionRun.getPhiStart()) xsDataExperimentalCondition.getGoniostat().setOscillationWidth(xsDataBestCollectionRun.getPhiWidth()) fRotationAxisEnd = ( xsDataBestCollectionRun.getPhiStart().getValue() + xsDataBestCollectionRun.getNumberOfImages().getValue() * xsDataBestCollectionRun.getPhiWidth().getValue() ) xsDataExperimentalCondition.getGoniostat().setRotationAxisEnd(XSDataAngle(fRotationAxisEnd)) xsDataSubWedge.setExperimentalCondition(xsDataExperimentalCondition) xsDataSubWedge.setSubWedgeNumber(xsDataBestCollectionRun.getCollectionRunNumber()) if xsDataBestCollectionRun.getCrystalPosition(): xsDataSubWedge.setAction( XSDataString("Crystal position: %d" % xsDataBestCollectionRun.getCrystalPosition().getValue()) ) else: xsDataSubWedge.setAction(xsDataBestCollectionRun.getAction()) xsDataCollectionStrategy.addSubWedge(xsDataSubWedge) xsDataCollectionStrategy.setSample(_xsDataSample) xsDataCollectionPlan.setCollectionStrategy(xsDataCollectionStrategy) xsDataStrategySummary = XSDataStrategySummary() xsDataStrategySummary.setCompleteness(xsDataBestStrategySummary.getCompleteness()) xsDataStrategySummary.setISigma(xsDataBestStrategySummary.getISigma()) xsDataStrategySummary.setRankingResolution(xsDataBestStrategySummary.getRankingResolution()) xsDataStrategySummary.setRedundancy(xsDataBestStrategySummary.getRedundancy()) xsDataStrategySummary.setResolution(xsDataBestStrategySummary.getResolution()) xsDataStrategySummary.setResolutionReasoning(xsDataBestStrategySummary.getResolutionReasoning()) xsDataStrategySummary.setTotalDataCollectionTime(xsDataBestStrategySummary.getTotalDataCollectionTime()) xsDataStrategySummary.setTotalExposureTime(xsDataBestStrategySummary.getTotalExposureTime()) xsDataCollectionPlan.setStrategySummary(xsDataStrategySummary) if xsDataBestCollectionPlan.getStatisticalPrediction() is not None: xsDataStatisticsStrategy = XSDataStatisticsStrategy.parseString( xsDataBestCollectionPlan.getStatisticalPrediction().marshal() ) xsDataCollectionPlan.setStatistics(xsDataStatisticsStrategy) xsDataCollectionPlan.setCollectionPlanNumber(xsDataBestCollectionPlan.getCollectionPlanNumber()) xsDataResultStrategy.addCollectionPlan(xsDataCollectionPlan) if _xsDataResultBest.getPathToLogFile() != None: xsDataResultStrategy.setBestLogFile(_xsDataResultBest.getPathToLogFile()) return xsDataResultStrategy