class EDPluginControlImageQualityIndicatorsv1_1(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__(self): EDPluginControl.__init__(self) self.strPluginExecWaitMultiFileName = "EDPluginWaitMultiFile" self.strPluginExecImageQualityIndicatorName = "EDPluginDistlSignalStrengthv1_1" self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = None self.xsDataResultControlImageQualityIndicators = None self.edPluginWaitMultiFile = None # Default time out for wait file self.fWaitFileTimeOut = 30 #s def checkParameters(self): """ Checks the mandatory parameters """ EDVerbose.DEBUG( "EDPluginControlImageQualityIndicatorsv1_1.checkParameters") self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlReadImageHeaderv10.configure") if self.getConfiguration(): xsDataStringParameter = EDConfiguration.getParamItem( self.getConfiguration(), "waitFileTimeOut") if xsDataStringParameter: self.fWaitFileTimeOut = float(xsDataStringParameter.getValue()) def preProcess(self, _edPlugin=None): """ Prepares the execution plugin """ EDPluginControl.preProcess(self, _edPlugin) EDVerbose.DEBUG( "EDPluginControlImageQualityIndicatorsv1_1.preProcess...") # List containing instances of all the exeuction plugins self.listPluginExecImageQualityIndicator = [] # Loop through all the incoming reference images self.edPluginWaitMultiFile = self.loadPlugin( self.strPluginExecWaitMultiFileName) listXSDataImage = self.getDataInput().getImage() xsDataInputWaitMultiFile = XSDataInputWaitMultiFile() for xsDataImage in listXSDataImage: xsDataInputWaitMultiFile.addExpectedFile( XSDataFile(xsDataImage.getPath())) xsDataInputWaitMultiFile.setExpectedSize(XSDataInteger(100000)) xsDataInputWaitMultiFile.setTimeOut(XSDataTime(self.fWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fWaitFileTimeOut) self.edPluginWaitMultiFile.setDataInput(xsDataInputWaitMultiFile) self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) EDVerbose.DEBUG("EDPluginControlImageQualityIndicatorsv1_1.process") self.edPluginWaitMultiFile.connectSUCCESS(self.doSuccessWaitMultiFile) self.edPluginWaitMultiFile.connectFAILURE(self.doFailureWaitMultiFile) self.executePluginSynchronous(self.edPluginWaitMultiFile) def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) EDVerbose.DEBUG( "EDPluginControlImageQualityIndicatorsv1_1.finallyProcess") self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def doSuccessWaitMultiFile(self, _edPlugin): """ The file has appeared on the disk """ self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_0.doSuccessWaitMultiFile") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlReadImageHeaderv10.doSuccessWaitMultiFile") # listXSDataImage = self.getDataInput().getImage() for (iIndex, xsDataImage) in enumerate(listXSDataImage): edPluginPluginExecImageQualityIndicator = self.loadPlugin(self.strPluginExecImageQualityIndicatorName, \ "%s-%d" % (self.strPluginExecImageQualityIndicatorName, iIndex + 1)) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength() xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) # Prepare the action cluster for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: edPluginPluginExecImageQualityIndicator.connectSUCCESS( self.doSuccessExecPlugin) edPluginPluginExecImageQualityIndicator.connectFAILURE( self.doFailureExecPlugin) self.addPluginToActionCluster( edPluginPluginExecImageQualityIndicator) # Launch the cluster self.executeActionCluster() self.synchronizeActionCluster() def doFailureWaitMultiFile(self, _edPlugin): """ The file has not appeared on the disk """ self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_0.doFailureWaitMultiFile") self.retrieveFailureMessages( _edPlugin, "EDPluginControlReadImageHeaderv10.doFailureWaitMultiFile") self.ERROR("Timeout when waiting for images") self.setFailure() def doSuccessExecPlugin(self, _edPlugin=None): EDVerbose.DEBUG( "EDPluginControlImageQualityIndicatorsv1_1.doSuccessExecPlugin") self.retrieveSuccessMessages( _edPlugin, "EDPluginControlImageQualityIndicatorsv1_1.doSuccessExecPlugin") xsDataImageQualityIndicators = _edPlugin.getDataOutput( ).getImageQualityIndicators() self.synchronizeOn() self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( XSDataImageQualityIndicators.parseString( xsDataImageQualityIndicators.marshal())) self.synchronizeOff() def doFailureExecPlugin(self, _edPlugin=None): EDVerbose.DEBUG( "EDPluginControlImageQualityIndicatorsv1_1.doFailureGeneratePrediction" ) #self.retrieveFailureMessages(_edPlugin, "EDPluginControlImageQualityIndicatorsv1_1.doFailureExecPlugin") def generateExecutiveSummary(self, _edPlugin=None): EDVerbose.DEBUG( "EDPluginControlImageQualityIndicatorsv1_1.generateExecutiveSummary" ) self.addErrorWarningMessagesToExecutiveSummary( "Image quality indicator plugin execution failure! Error messages: " ) self.addExecutiveSummaryLine( "Summary of image quality indicators with %s :" % self.strPluginExecImageQualityIndicatorName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary( edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False)
class EDPluginControlImageQualityIndicatorsv1_2(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__(self): EDPluginControl.__init__(self) self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1" self.strPluginName = "EDPluginDistlSignalStrengthv1_1" self.strPluginNameThinClient = "EDPluginDistlSignalStrengthv1_1" self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_4" self.strIndexingMOSFLMPluginName = "EDPluginMOSFLMIndexingv10" self.edPluginMOSFLMIndexing = None self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.edPluginReadImageHeader = None self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = [] self.xsDataResultControlImageQualityIndicators = None self.edPluginMXWaitFile = None # Default time out for wait file self.fMXWaitFileTimeOut = 30 #s # Flag for using the thin client self.bUseThinClient = True self.edPluginISPyB = None self.listPluginMOSFLM = [] self.bDoISPyBUpload = True self.defaultMinImageSize = 1000000 self.minImageSize = None def checkParameters(self): """ Checks the mandatory parameters """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.checkParameters") self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlReadImageHeaderv10.configure") self.fMXWaitFileTimeOut = float( self.config.get("MXWaitFileTimeOut", self.fMXWaitFileTimeOut)) self.bDoISPyBUpload = self.config.get("do_ispyb_upload") if self.bDoISPyBUpload is None or self.bDoISPyBUpload == "false": self.bDoISPyBUpload = False self.minImageSize = self.config.get("minImageSize") if self.minImageSize is None: self.minImageSize = self.defaultMinImageSize def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.process") EDUtilsParallel.initializeNbThread() # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPlugin = [] for xsDataImage in listXSDataImage: self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): strError = "Time-out while waiting for image %s" % xsDataImage.path.value self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) listPlugin.append(edPluginPluginExecImageQualityIndicator) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for edPluginPluginExecImageQualityIndicator in listPlugin: edPluginPluginExecImageQualityIndicator.synchronize() xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString( \ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.bDoISPyBUpload: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key= lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value ) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection( self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing( xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin( self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([ edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator ]) edPluginMOSFLMIndexing.setDataInput( xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult( xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution # print xsDataIndexingResult.marshal() # xsDataResultISPyB = edPluginISPyB.dataOutput # if xsDataResultISPyB is not None: #print xsDataResultISPyB.marshal() def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) # Synchronize ISPyB plugin self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.finallyProcess") if self.bDoISPyBUpload: self.edPluginISPyB.synchronize() listId = [] for xsDataInteger in self.edPluginISPyB.dataOutput.imageQualityIndicatorsId: listId.append(xsDataInteger.value) self.DEBUG("ISPyB imageQualityIndicatorIds = %r" % listId) self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def generateExecutiveSummary(self, _edPlugin=None): self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_2.generateExecutiveSummary" ) self.addErrorWarningMessagesToExecutiveSummary( "Image quality indicator plugin execution failure! Error messages: " ) self.addExecutiveSummaryLine( "Summary of image quality indicators with %s :" % self.strPluginName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary( edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False)
class EDPluginControlImageQualityIndicatorsv1_3(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__(self): EDPluginControl.__init__(self) self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1" self.strPluginName = "EDPluginDistlSignalStrengthv1_1" self.strPluginNameThinClient = "EDPluginDistlSignalStrengthThinClientv1_1" self.strPluginNameControlBackground3D = "EDPluginControlBackground3Dv1_0" self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_4" self.strIndexingMOSFLMPluginName = "EDPluginMOSFLMIndexingv10" self.edPluginMOSFLMIndexing = None self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.edPluginReadImageHeader = None self.edPluginControlBackground3D = None self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = [] self.listPluginControlBackground3D = [] self.xsDataResultControlImageQualityIndicators = None self.edPluginMXWaitFile = None # Default time out for wait file self.fMXWaitFileTimeOut = 30 # s # Flag for using the thin client - disabled as of 2016/07/20 self.bUseThinClient = False self.edPluginISPyB = None self.listPluginMOSFLM = [] self.defaultMinImageSize = 1000000 self.minImageSize = None def checkParameters(self): """ Checks the mandatory parameters """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_3.checkParameters") self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlReadImageHeaderv10.configure") self.fMXWaitFileTimeOut = float( self.config.get("MXWaitFileTimeOut", self.fMXWaitFileTimeOut)) self.minImageSize = self.config.get("minImageSize") if self.minImageSize is None: self.minImageSize = self.defaultMinImageSize def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_3.process") EDUtilsParallel.initializeNbThread() # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPlugin = [] for xsDataImage in listXSDataImage: self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(5000000)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): strError = "Time-out while waiting for image %s" % xsDataImage.path.value self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() edPluginControlBackground3D = self.loadPlugin( self.strPluginNameControlBackground3D) listPlugin.append([ edPluginPluginExecImageQualityIndicator, edPluginControlBackground3D ]) xsDataInputControlBackground3D = XSDataInputControlBackground3D( ) xsDataInputControlBackground3D.addImage( XSDataFile(xsDataImage.path)) edPluginControlBackground3D.dataInput = xsDataInputControlBackground3D edPluginControlBackground3D.execute() listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for pluginPair in listPlugin: edPluginPluginExecImageQualityIndicator = pluginPair[0] edPluginControlBackground3D = pluginPair[1] edPluginPluginExecImageQualityIndicator.synchronize() edPluginControlBackground3D.synchronize() xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) if edPluginControlBackground3D.dataOutput.imageBackground != []: xsDataImageQualityIndicators.background3D_estimate = edPluginControlBackground3D.dataOutput.imageBackground[ 0].estimate self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have background3D_estimate from all images: has_background3D_estimate = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.background3D_estimate is None: has_background3D_estimate = False if has_background3D_estimate: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators. background3D_estimate.value) else: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators. totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection( self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing( xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin( self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([ edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator ]) edPluginMOSFLMIndexing.setDataInput( xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult( xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution # print xsDataIndexingResult.marshal() # xsDataResultISPyB = edPluginISPyB.dataOutput # if xsDataResultISPyB is not None: # print xsDataResultISPyB.marshal() def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) if self.edPluginISPyB is not None: # Synchronize ISPyB plugin self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_3.finallyProcess") self.edPluginISPyB.synchronize() listId = [] for xsDataInteger in self.edPluginISPyB.dataOutput.imageQualityIndicatorsId: listId.append(xsDataInteger.value) self.DEBUG("ISPyB imageQualityIndicatorIds = %r" % listId) self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def generateExecutiveSummary(self, _edPlugin=None): self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_3.generateExecutiveSummary" ) self.addErrorWarningMessagesToExecutiveSummary( "Image quality indicator plugin execution failure! Error messages: " ) self.addExecutiveSummaryLine( "Summary of image quality indicators with %s :" % self.strPluginName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary( edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False)
class EDPluginControlImageQualityIndicatorsv1_5(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__ (self): EDPluginControl.__init__(self) self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1" self.strPluginName = "EDPluginDistlSignalStrengthv1_1" self.strPluginNameThinClient = "EDPluginDistlSignalStrengthThinClientv1_1" self.strPluginNameControlDozor = "EDPluginControlDozorv1_0" self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_5" self.strIndexingLabelitPluginName = "EDPluginLabelitIndexingv1_1" self.edPluginLabelitIndexing = None self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.edPluginReadImageHeader = None self.edPluginControlDozor = None self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = [] self.listPluginControlDozor = [] self.xsDataResultControlImageQualityIndicators = None self.edPluginMXWaitFile = None # Default time out for wait file self.fMXWaitFileTimeOut = 30 # s # Flag for using the thin client if EDUtilsPath.isEMBL(): self.bUseThinClient = False else: self.bUseThinClient = True self.edPluginISPyB = None self.listPluginLabelit = [] self.defaultMinImageSize = 1000000 self.minImageSize = None def checkParameters(self): """ Checks the mandatory parameters """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.checkParameters") self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlReadImageHeaderv10.configure") self.fMXWaitFileTimeOut = float(self.config.get("MXWaitFileTimeOut", self.fMXWaitFileTimeOut)) self.minImageSize = self.config.get("minImageSize") if self.minImageSize is None: self.minImageSize = self.defaultMinImageSize def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators() listPluginDistl = [] listPluginDozor = [] listBatch = [] indexBatch = 0 for xsDataImage in listXSDataImage: self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path) xsDataInputMXWaitFile.setSize(XSDataInteger(5000000)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(xsDataImage.path.value): strError = "Time-out while waiting for image %s" % xsDataImage.path.value self.error(strError) self.addErrorMessage(strError) self.setFailure() else: # Check if we should run distl.signalStrength edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName) self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength() xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append((xsDataImage.copy(), edPluginPluginExecImageQualityIndicator)) listBatch.append(xsDataImage.copy()) if len(listBatch) == batchSize: edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append((edPluginControlDozor, listBatch)) listBatch = [] if len(listBatch) > 0: # Process the remaining images... edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append([edPluginControlDozor, listBatch]) listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators() for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString( edPluginControlDozor.dataOutput.inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: edPluginLabelitIndexing = self.loadPlugin(self.strIndexingLabelitPluginName) xsDataInputLabelitIndexing = XSDataInputLabelitIndexing() xsDataInputLabelitIndexing.image.append(XSDataImage(xsDataResultControlImageQualityIndicator.image.path)) edPluginLabelitIndexing.setDataInput(xsDataInputLabelitIndexing) self.listPluginLabelit.append([edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator]) edPluginLabelitIndexing.execute() for tupleLabelit in self.listPluginLabelit: edPluginLabelitIndexing = tupleLabelit[0] xsDataResultControlImageQualityIndicator = tupleLabelit[1] edPluginLabelitIndexing.synchronize() if not edPluginLabelitIndexing.isFailure() and edPluginLabelitIndexing.dataOutput is not None: xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput() xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(xsDataResultLabelitIndexing) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) if self.edPluginISPyB is not None: # Synchronize ISPyB plugin self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.finallyProcess") self.edPluginISPyB.synchronize() listId = [] for xsDataInteger in self.edPluginISPyB.dataOutput.imageQualityIndicatorsId: listId.append(xsDataInteger.value) self.DEBUG("ISPyB imageQualityIndicatorIds = %r" % listId) self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def generateExecutiveSummary(self, _edPlugin=None): self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.generateExecutiveSummary") self.addErrorWarningMessagesToExecutiveSummary("Image quality indicator plugin execution failure! Error messages: ") self.addExecutiveSummaryLine("Summary of image quality indicators with %s :" % self.strPluginName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary(edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False)
class EDPluginControlImageQualityIndicatorsv1_4(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__ (self): EDPluginControl.__init__(self) self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1" self.strPluginName = "EDPluginDistlSignalStrengthv1_1" self.strPluginNameThinClient = "EDPluginDistlSignalStrengthThinClientv1_1" self.strPluginNameControlDozor = "EDPluginControlDozorv1_0" self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_4" self.strIndexingMOSFLMPluginName = "EDPluginMOSFLMIndexingv10" self.edPluginMOSFLMIndexing = None self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.edPluginReadImageHeader = None self.edPluginControlDozor = None self.strPluginControlH5ToCBF = "EDPluginControlH5ToCBFv1_1" self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = [] self.listPluginControlDozor = [] self.xsDataResultControlImageQualityIndicators = None self.edPluginMXWaitFile = None # Default time out for wait file self.fMXWaitFileTimeOut = 30 # s # Flag for using the thin client self.bUseThinClient = True self.edPluginISPyB = None self.listPluginMOSFLM = [] self.defaultMinImageSize = 1000000 self.minImageSize = None def checkParameters(self): """ Checks the mandatory parameters """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.checkParameters") self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.configure") self.fMXWaitFileTimeOut = float(self.config.get("MXWaitFileTimeOut", self.fMXWaitFileTimeOut)) self.minImageSize = self.config.get("minImageSize") if self.minImageSize is None: self.minImageSize = self.defaultMinImageSize def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value self.screen("Batch size: {0}".format(batchSize)) # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Loop through all the incoming reference images listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators() listPluginDistl = [] listPluginDozor = [] listBatch = [] indexBatch = 0 listH5FilePath = [] ispybDataCollection = None for xsDataImage in listXSDataImage: strPathToImage = xsDataImage.path.value # If Eiger, just wait for the h5 file if "id30a3" in strPathToImage: h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage, batchSize) # print(h5FilePath) # print(hdf5ImageNumber) if not h5DataFilePath in listH5FilePath: self.screen("ID30a3 Eiger data, waiting for master and data files...") listH5FilePath.append(h5DataFilePath) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Waiting for file {0}".format(h5DataFilePath)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() # hdf5FilePath = strPathToImage.replace(".cbf", ".h5") ispybDataCollection = None time.sleep(1) indexLoop = 1 continueLoop = True while continueLoop: xsDataInputControlH5ToCBF = XSDataInputControlH5ToCBF() xsDataInputControlH5ToCBF.hdf5File = XSDataFile(XSDataString(strPathToImage)) imageNumber = EDUtilsImage.getImageNumber(strPathToImage) xsDataInputControlH5ToCBF.imageNumber = XSDataInteger(imageNumber) xsDataInputControlH5ToCBF.hdf5ImageNumber = XSDataInteger(hdf5ImageNumber) xsDataInputControlH5ToCBF.ispybDataCollection = ispybDataCollection edPluginControlH5ToCBF = self.loadPlugin(self.strPluginControlH5ToCBF, "ControlH5ToCBF_%04d_%02d" % (imageNumber, indexLoop)) edPluginControlH5ToCBF.dataInput = xsDataInputControlH5ToCBF edPluginControlH5ToCBF.executeSynchronous() cbfFile = edPluginControlH5ToCBF.dataOutput.outputCBFFile # print(cbfFile) # print(indexLoop) if cbfFile is not None: strPathToImage = cbfFile.path.value # print(cbfFile.path.value) if os.path.exists(strPathToImage): self.screen("Image has been converted to CBF file: {0}".format(strPathToImage)) continueLoop = False # print(continueLoop) if continueLoop: self.screen("Still waiting for converting to CBF file: {0}".format(strPathToImage)) indexLoop += 1 time.sleep(5) if indexLoop > 10: continueLoop = False ispybDataCollection = edPluginControlH5ToCBF.dataOutput.ispybDataCollection elif not os.path.exists(strPathToImage): self.screen("Waiting for file {0}".format(strPathToImage)) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(strPathToImage): strError = "Time-out while waiting for image %s" % strPathToImage self.error(strError) self.addErrorMessage(strError) self.setFailure() else: # Check if we should run distl.signalStrength xsDataImageNew = XSDataImage(XSDataString(strPathToImage)) edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName) self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength() xsDataInputDistlSignalStrength.setReferenceImage(xsDataImageNew) edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append((xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator)) listBatch.append(xsDataImageNew.copy()) if len(listBatch) == batchSize: edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append((edPluginControlDozor, listBatch)) listBatch = [] if len(listBatch) > 0: # Process the remaining images... edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append([edPluginControlDozor, listBatch]) listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators() for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile if imageDozor.spotFile is not None: if os.path.exists(imageDozor.spotFile.path.value): numpyArray = numpy.loadtxt(imageDozor.spotFile.path.value, skiprows=3) xsDataImageQualityIndicators.dozorSpotList = XSDataString(base64.b64encode(numpyArray.tostring())) xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[0])) if len(numpyArray.shape) > 1: xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[1])) xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString( edPluginControlDozor.dataOutput.inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: xsDataSubWedge = xsDataResultReadImageHeader.subWedge self.xsDataCollection = XSDataCollection() self.xsDataCollection.addSubWedge(xsDataSubWedge) xsDataIndexingInput = XSDataIndexingInput() xsDataIndexingInput.setDataCollection(self.xsDataCollection) xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(xsDataIndexingInput) edPluginMOSFLMIndexing = self.loadPlugin(self.strIndexingMOSFLMPluginName) self.listPluginMOSFLM.append([edPluginMOSFLMIndexing, xsDataResultControlImageQualityIndicator]) edPluginMOSFLMIndexing.setDataInput(xsDataMOSFLMIndexingInput) edPluginMOSFLMIndexing.execute() for tupleMOSFLM in self.listPluginMOSFLM: edPluginMOSFLMIndexing = tupleMOSFLM[0] xsDataResultControlImageQualityIndicator = tupleMOSFLM[1] edPluginMOSFLMIndexing.synchronize() if not edPluginMOSFLMIndexing.isFailure(): xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(xsDataMOSFLMOutput) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution # print xsDataIndexingResult.marshal() # xsDataResultISPyB = edPluginISPyB.dataOutput # if xsDataResultISPyB is not None: # print xsDataResultISPyB.marshal() def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) if self.edPluginISPyB is not None: # Synchronize ISPyB plugin self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.finallyProcess") self.edPluginISPyB.synchronize() listId = [] for xsDataInteger in self.edPluginISPyB.dataOutput.imageQualityIndicatorsId: listId.append(xsDataInteger.value) self.DEBUG("ISPyB imageQualityIndicatorIds = %r" % listId) self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def generateExecutiveSummary(self, _edPlugin=None): self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.generateExecutiveSummary") self.addErrorWarningMessagesToExecutiveSummary("Image quality indicator plugin execution failure! Error messages: ") self.addExecutiveSummaryLine("Summary of image quality indicators with %s :" % self.strPluginName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary(edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False) def getH5FilePath(self, filePath, batchSize=1): imageNumber = EDUtilsImage.getImageNumber(filePath) prefix = EDUtilsImage.getPrefix(filePath) h5FileNumber = int((imageNumber - 1) / batchSize) * batchSize + 1 h5MasterFileName = "{prefix}_{h5FileNumber}_master.h5".format(prefix=prefix, h5FileNumber=h5FileNumber) h5MasterFilePath = os.path.join(os.path.dirname(filePath), h5MasterFileName) h5DataFileName = "{prefix}_{h5FileNumber}_data_000001.h5".format(prefix=prefix, h5FileNumber=h5FileNumber) h5DataFilePath = os.path.join(os.path.dirname(filePath), h5DataFileName) return h5MasterFilePath, h5DataFilePath, h5FileNumber
class EDPluginControlImageQualityIndicatorsv1_5(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__(self): EDPluginControl.__init__(self) self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1" self.strPluginName = "EDPluginDistlSignalStrengthv1_1" self.strPluginNameThinClient = "EDPluginDistlSignalStrengthThinClientv1_1" self.strPluginNameControlDozor = "EDPluginControlDozorv1_0" self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_5" self.strIndexingLabelitPluginName = "EDPluginLabelitIndexingv1_1" self.edPluginLabelitIndexing = None self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.edPluginReadImageHeader = None self.edPluginControlDozor = None self.strPluginControlH5ToCBF = "EDPluginControlH5ToCBFv1_1" self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = [] self.listPluginControlDozor = [] self.xsDataResultControlImageQualityIndicators = None self.edPluginMXWaitFile = None # Default time out for wait file self.fMXWaitFileTimeOut = 120 # s # Flag for using the thin client - disabled as of 2016/07/20 self.bUseThinClient = False self.edPluginISPyB = None self.listPluginLabelit = [] self.defaultMinImageSize = 1000000 self.minImageSize = None self.strEDPluginControlReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.hasOverlap = False def checkParameters(self): """ Checks the mandatory parameters """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.checkParameters") # self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlReadImageHeaderv10.configure") self.fMXWaitFileTimeOut = float( self.config.get("MXWaitFileTimeOut", self.fMXWaitFileTimeOut)) self.minImageSize = self.config.get("minImageSize") if self.minImageSize is None: self.minImageSize = self.defaultMinImageSize def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value self.screen( "Image quality indicators batch size: {0}".format(batchSize)) # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Check if fast mesh (for HDF5) isFastMesh = False if self.dataInput.fastMesh: isFastMesh = self.dataInput.fastMesh.value # Loop through all the incoming reference images if len(self.dataInput.image) == 0: directory = self.dataInput.directory.path.value template = self.dataInput.template.value startNo = self.dataInput.startNo.value endNo = self.dataInput.endNo.value listXSDataImage = [] for index in range(startNo, endNo + 1): imageName = template.replace("####", "{0:04d}".format(index)) imagePath = os.path.join(directory, imageName) xsDataImage = XSDataImage(path=XSDataString(imagePath), number=XSDataInteger(index)) listXSDataImage.append(xsDataImage) else: listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators( ) listPluginDistl = [] listPluginDozor = [] listOfImagesInBatch = [] listOfAllBatches = [] indexBatch = 0 listH5FilePath = [] # Process data in batches for xsDataImage in listXSDataImage: listOfImagesInBatch.append(xsDataImage.copy()) if len(listOfImagesInBatch) == batchSize: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] if len(listOfImagesInBatch) > 0: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] # Loop over batches for listOfImagesInBatch in listOfAllBatches: # First wait for images for image in listOfImagesInBatch: strPathToImage = image.path.value # If Eiger, just wait for the h5 file if strPathToImage.endswith(".h5"): h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath( strPathToImage, batchSize=batchSize, isFastMesh=isFastMesh) # print(h5FilePath) # print(hdf5ImageNumber) if not h5DataFilePath in listH5FilePath: self.screen( "ID30a3 Eiger data, waiting for master and data files..." ) listH5FilePath.append(h5DataFilePath) self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile( XSDataString(h5DataFilePath)) xsDataInputMXWaitFile.setSize( XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.screen( "Waiting for file {0}".format(h5DataFilePath)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput( xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() # hdf5FilePath = strPathToImage.replace(".cbf", ".h5") time.sleep(1) if not os.path.exists(h5DataFilePath): strError = "Time-out while waiting for image %s" % h5DataFilePath self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if not os.path.exists(strPathToImage): # self.screen("Waiting for file {0}".format(strPathToImage)) self.edPluginMXWaitFile = self.loadPlugin( self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile( XSDataString(strPathToImage)) xsDataInputMXWaitFile.setSize( XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut( XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Wait file timeOut set to %.0f s" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput( xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(strPathToImage): strError = "Time-out while waiting for image %s" % strPathToImage self.error(strError) self.addErrorMessage(strError) self.setFailure() if not self.isFailure(): strPathToFirstImage = listOfImagesInBatch[0].path.value if strPathToImage.endswith(".h5"): indexLoop = 1 continueLoop = True while continueLoop: directory = os.path.dirname(strPathToFirstImage) firstImage = EDUtilsImage.getImageNumber( listOfImagesInBatch[0].path.value) lastImage = EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value) xsDataInputH5ToCBF = XSDataInputH5ToCBF() xsDataInputH5ToCBF.hdf5File = XSDataFile( listOfImagesInBatch[0].path) xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1) xsDataInputH5ToCBF.startImageNumber = XSDataInteger( firstImage) xsDataInputH5ToCBF.endImageNumber = XSDataInteger( lastImage) xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile( XSDataString(directory)) edPluginH5ToCBF = self.loadPlugin( "EDPluginH5ToCBFv1_1") edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF edPluginH5ToCBF.execute() edPluginH5ToCBF.synchronize() outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate if outputCBFFileTemplate is not None: lastCbfFile = outputCBFFileTemplate.path.value.replace( "######", "{0:06d}".format( EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value))) strPathToImage = os.path.join( directory, lastCbfFile) # print(cbfFile.path.value) if os.path.exists(strPathToImage): # Rename all images for image in listOfImagesInBatch: image.path.value = image.path.value.replace( ".h5", ".cbf") imageNumber = EDUtilsImage.getImageNumber( image.path.value) oldPath = os.path.join( directory, outputCBFFileTemplate.path.value. replace("######", "{0:06d}".format(imageNumber))) newPath = os.path.join( directory, outputCBFFileTemplate.path.value. replace("######", "{0:04d}".format(imageNumber))) os.rename(oldPath, newPath) lastCbfFile = outputCBFFileTemplate.path.value.replace( "######", "{0:04d}".format( EDUtilsImage.getImageNumber( listOfImagesInBatch[-1].path.value) )) strPathToImage = os.path.join( directory, lastCbfFile) self.screen( "Image has been converted to CBF file: {0}" .format(strPathToImage)) continueLoop = False # print(continueLoop) if continueLoop: self.screen( "Still waiting for converting to CBF file: {0}" .format(strPathToImage)) indexLoop += 1 time.sleep(5) if indexLoop > 10: continueLoop = False for image in listOfImagesInBatch: strPathToImage = image.path.value # Check if we should run distl.signalStrength xsDataImageNew = XSDataImage(XSDataString(strPathToImage)) xsDataImageNew.number = XSDataInteger( EDUtilsImage.getImageNumber(image.path.value)) edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin( strPluginName) self.listPluginExecImageQualityIndicator.append( edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength( ) xsDataInputDistlSignalStrength.setReferenceImage( xsDataImageNew) edPluginPluginExecImageQualityIndicator.setDataInput( xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append( (xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator)) edPluginControlDozor = self.loadPlugin( self.strPluginNameControlDozor, "ControlDozor_{0}".format( os.path.splitext( os.path.basename(strPathToFirstImage))[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listOfImagesInBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger( len(listOfImagesInBatch)) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append( (edPluginControlDozor, list(listOfImagesInBatch))) if not self.isFailure(): listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators( ) for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput is not None: if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators( xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() # Check that we got at least one result if len(edPluginControlDozor.dataOutput.imageDozor) == 0: # Run the dozor plugin again, this time synchronously firstImage = os.path.basename(listBatch[0].path.value) lastImage = os.path.basename(listBatch[-1].path.value) self.screen( "No dozor results! Re-executing Dozor for images {0} to {1}" .format(firstImage, lastImage)) time.sleep(5) edPluginControlDozor = self.loadPlugin( self.strPluginNameControlDozor, "ControlDozor_reexecution_{0}".format( os.path.splitext(firstImage)[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile( image.path)) xsDataInputControlDozor.batchSize = XSDataInteger( batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.executeSynchronous() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile if imageDozor.spotFile is not None: if os.path.exists( imageDozor.spotFile.path.value): numpyArray = numpy.loadtxt( imageDozor.spotFile.path.value, skiprows=3) xsDataImageQualityIndicators.dozorSpotList = XSDataString( base64.b64encode( numpyArray.tostring())) xsDataImageQualityIndicators.addDozorSpotListShape( XSDataInteger(numpyArray.shape[0])) if len(numpyArray.shape) > 1: xsDataImageQualityIndicators.addDozorSpotListShape( XSDataInteger(numpyArray.shape[1])) xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput( ).parseString( edPluginControlDozor.dataOutput. inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators( xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted( self.xsDataResultControlImageQualityIndicators. imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[ -5:]: if xsDataResultControlImageQualityIndicator.dozor_score.value > 1: xsDataInputReadImageHeader = XSDataInputReadImageHeader( ) xsDataInputReadImageHeader.image = XSDataFile( xsDataResultControlImageQualityIndicator.image.path ) self.edPluginReadImageHeader = self.loadPlugin( self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: edPluginLabelitIndexing = self.loadPlugin( self.strIndexingLabelitPluginName) xsDataInputLabelitIndexing = XSDataInputLabelitIndexing( ) xsDataInputLabelitIndexing.image.append( XSDataImage( xsDataResultControlImageQualityIndicator. image.path)) edPluginLabelitIndexing.setDataInput( xsDataInputLabelitIndexing) self.listPluginLabelit.append([ edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator ]) edPluginLabelitIndexing.execute() for tupleLabelit in self.listPluginLabelit: edPluginLabelitIndexing = tupleLabelit[0] xsDataResultControlImageQualityIndicator = tupleLabelit[1] edPluginLabelitIndexing.synchronize() if not edPluginLabelitIndexing.isFailure( ) and edPluginLabelitIndexing.dataOutput is not None: xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput( ) xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult( xsDataResultLabelitIndexing) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) if self.edPluginISPyB is not None: # Synchronize ISPyB plugin self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_5.finallyProcess") self.edPluginISPyB.synchronize() listId = [] for xsDataInteger in self.edPluginISPyB.dataOutput.imageQualityIndicatorsId: listId.append(xsDataInteger.value) self.DEBUG("ISPyB imageQualityIndicatorIds = %r" % listId) self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def generateExecutiveSummary(self, _edPlugin=None): self.DEBUG( "EDPluginControlImageQualityIndicatorsv1_5.generateExecutiveSummary" ) self.addErrorWarningMessagesToExecutiveSummary( "Image quality indicator plugin execution failure! Error messages: " ) self.addExecutiveSummaryLine( "Summary of image quality indicators with %s :" % self.strPluginName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary( edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False) def getH5FilePath(self, filePath, batchSize=1, isFastMesh=False): imageNumber = EDUtilsImage.getImageNumber(filePath) prefix = EDUtilsImage.getPrefix(filePath) if isFastMesh: h5ImageNumber = int((imageNumber - 1) / 100) + 1 h5FileNumber = 1 else: h5ImageNumber = 1 h5FileNumber = int((imageNumber - 1) / batchSize) * batchSize + 1 h5MasterFileName = "{prefix}_{h5FileNumber}_master.h5".format( prefix=prefix, h5FileNumber=h5FileNumber) h5MasterFilePath = os.path.join(os.path.dirname(filePath), h5MasterFileName) h5DataFileName = "{prefix}_{h5FileNumber}_data_{h5ImageNumber:06d}.h5".format( prefix=prefix, h5FileNumber=h5FileNumber, h5ImageNumber=h5ImageNumber) h5DataFilePath = os.path.join(os.path.dirname(filePath), h5DataFileName) return h5MasterFilePath, h5DataFilePath, h5FileNumber
class EDPluginControlImageQualityIndicatorsv1_1(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__ (self): EDPluginControl.__init__(self) self.strPluginExecWaitMultiFileName = "EDPluginWaitMultiFile" self.strPluginExecImageQualityIndicatorName = "EDPluginDistlSignalStrengthv1_1" self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = None self.xsDataResultControlImageQualityIndicators = None self.edPluginWaitMultiFile = None # Default time out for wait file self.fWaitFileTimeOut = 30 #s def checkParameters(self): """ Checks the mandatory parameters """ EDVerbose.DEBUG("EDPluginControlImageQualityIndicatorsv1_1.checkParameters") self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin = None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlReadImageHeaderv10.configure") if self.getConfiguration(): xsDataStringParameter = EDConfiguration.getParamItem(self.getConfiguration(), "waitFileTimeOut") if xsDataStringParameter: self.fWaitFileTimeOut = float(xsDataStringParameter.getValue()) def preProcess(self, _edPlugin=None): """ Prepares the execution plugin """ EDPluginControl.preProcess(self, _edPlugin) EDVerbose.DEBUG("EDPluginControlImageQualityIndicatorsv1_1.preProcess...") # List containing instances of all the exeuction plugins self.listPluginExecImageQualityIndicator = [] # Loop through all the incoming reference images self.edPluginWaitMultiFile = self.loadPlugin(self.strPluginExecWaitMultiFileName) listXSDataImage = self.getDataInput().getImage() xsDataInputWaitMultiFile = XSDataInputWaitMultiFile() for xsDataImage in listXSDataImage: xsDataInputWaitMultiFile.addExpectedFile(XSDataFile(xsDataImage.getPath())) xsDataInputWaitMultiFile.setExpectedSize(XSDataInteger(100000)) xsDataInputWaitMultiFile.setTimeOut(XSDataTime(self.fWaitFileTimeOut)) self.DEBUG("Wait file timeOut set to %f" % self.fWaitFileTimeOut) self.edPluginWaitMultiFile.setDataInput(xsDataInputWaitMultiFile) self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators() def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) EDVerbose.DEBUG("EDPluginControlImageQualityIndicatorsv1_1.process") self.edPluginWaitMultiFile.connectSUCCESS(self.doSuccessWaitMultiFile) self.edPluginWaitMultiFile.connectFAILURE(self.doFailureWaitMultiFile) self.executePluginSynchronous(self.edPluginWaitMultiFile) def finallyProcess(self, _edPlugin= None): EDPluginControl.finallyProcess(self, _edPlugin) EDVerbose.DEBUG("EDPluginControlImageQualityIndicatorsv1_1.finallyProcess") self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def doSuccessWaitMultiFile(self, _edPlugin): """ The file has appeared on the disk """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_0.doSuccessWaitMultiFile") self.retrieveSuccessMessages(_edPlugin, "EDPluginControlReadImageHeaderv10.doSuccessWaitMultiFile") # listXSDataImage = self.getDataInput().getImage() for (iIndex, xsDataImage) in enumerate(listXSDataImage): edPluginPluginExecImageQualityIndicator = self.loadPlugin(self.strPluginExecImageQualityIndicatorName, \ "%s-%d" % (self.strPluginExecImageQualityIndicatorName, iIndex + 1)) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength() xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage) edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength) self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator) # Prepare the action cluster for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: edPluginPluginExecImageQualityIndicator.connectSUCCESS(self.doSuccessExecPlugin) edPluginPluginExecImageQualityIndicator.connectFAILURE(self.doFailureExecPlugin) self.addPluginToActionCluster(edPluginPluginExecImageQualityIndicator) # Launch the cluster self.executeActionCluster() self.synchronizeActionCluster() def doFailureWaitMultiFile(self, _edPlugin): """ The file has not appeared on the disk """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_0.doFailureWaitMultiFile") self.retrieveFailureMessages(_edPlugin, "EDPluginControlReadImageHeaderv10.doFailureWaitMultiFile") self.ERROR("Timeout when waiting for images") self.setFailure() def doSuccessExecPlugin(self, _edPlugin=None): EDVerbose.DEBUG("EDPluginControlImageQualityIndicatorsv1_1.doSuccessExecPlugin") self.retrieveSuccessMessages(_edPlugin, "EDPluginControlImageQualityIndicatorsv1_1.doSuccessExecPlugin") xsDataImageQualityIndicators = _edPlugin.getDataOutput().getImageQualityIndicators() self.synchronizeOn() self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(XSDataImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())) self.synchronizeOff() def doFailureExecPlugin(self, _edPlugin=None): EDVerbose.DEBUG("EDPluginControlImageQualityIndicatorsv1_1.doFailureGeneratePrediction") #self.retrieveFailureMessages(_edPlugin, "EDPluginControlImageQualityIndicatorsv1_1.doFailureExecPlugin") def generateExecutiveSummary(self, _edPlugin=None): EDVerbose.DEBUG("EDPluginControlImageQualityIndicatorsv1_1.generateExecutiveSummary") self.addErrorWarningMessagesToExecutiveSummary("Image quality indicator plugin execution failure! Error messages: ") self.addExecutiveSummaryLine("Summary of image quality indicators with %s :" % self.strPluginExecImageQualityIndicatorName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary(edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False)
class EDPluginControlImageQualityIndicatorsv1_5(EDPluginControl): """ This plugin that control the plugin that generates the image quality indicators. """ def __init__ (self): EDPluginControl.__init__(self) self.strPluginMXWaitFileName = "EDPluginMXWaitFilev1_1" self.strPluginName = "EDPluginDistlSignalStrengthv1_1" self.strPluginNameThinClient = "EDPluginDistlSignalStrengthThinClientv1_1" self.strPluginNameControlDozor = "EDPluginControlDozorv1_0" self.strISPyBPluginName = "EDPluginISPyBStoreListOfImageQualityIndicatorsv1_5" self.strIndexingLabelitPluginName = "EDPluginLabelitIndexingv1_1" self.edPluginLabelitIndexing = None self.strPluginReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.edPluginReadImageHeader = None self.edPluginControlDozor = None self.strPluginControlH5ToCBF = "EDPluginControlH5ToCBFv1_1" self.setXSDataInputClass(XSDataInputControlImageQualityIndicators) self.listPluginExecImageQualityIndicator = [] self.listPluginControlDozor = [] self.xsDataResultControlImageQualityIndicators = None self.edPluginMXWaitFile = None # Default time out for wait file self.fMXWaitFileTimeOut = 120 # s # Flag for using the thin client - disabled as of 2016/07/20 self.bUseThinClient = False self.edPluginISPyB = None self.listPluginLabelit = [] self.defaultMinImageSize = 1000000 self.minImageSize = None self.strEDPluginControlReadImageHeaderName = "EDPluginControlReadImageHeaderv10" self.hasOverlap = False def checkParameters(self): """ Checks the mandatory parameters """ self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.checkParameters") # self.checkMandatoryParameters(self.getDataInput().getImage(), "Image") def configure(self, _edPlugin=None): EDPluginControl.configure(self) self.DEBUG("EDPluginControlReadImageHeaderv10.configure") self.fMXWaitFileTimeOut = float(self.config.get("MXWaitFileTimeOut", self.fMXWaitFileTimeOut)) self.minImageSize = self.config.get("minImageSize") if self.minImageSize is None: self.minImageSize = self.defaultMinImageSize def process(self, _edPlugin=None): """ Executes the execution plugins """ EDPluginControl.process(self, _edPlugin) self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process") EDUtilsParallel.initializeNbThread() # Check batch size if self.dataInput.batchSize is None: batchSize = 1 else: batchSize = self.dataInput.batchSize.value self.screen("Image quality indicators batch size: {0}".format(batchSize)) # Check if we should do distlSignalStrength: bDoDistlSignalStrength = True if self.dataInput.doDistlSignalStrength is not None: if not self.dataInput.doDistlSignalStrength.value: bDoDistlSignalStrength = False # Check if we should do indexing: bDoIndexing = False if self.dataInput.doIndexing is not None: if self.dataInput.doIndexing.value: bDoIndexing = True # Check if fast mesh (for HDF5) isFastMesh = False if self.dataInput.fastMesh: isFastMesh = self.dataInput.fastMesh.value # Loop through all the incoming reference images if len(self.dataInput.image) == 0: directory = self.dataInput.directory.path.value template = self.dataInput.template.value startNo = self.dataInput.startNo.value endNo = self.dataInput.endNo.value listXSDataImage = [] for index in range(startNo, endNo + 1): imageName = template.replace("####", "{0:04d}".format(index)) imagePath = os.path.join(directory, imageName) xsDataImage = XSDataImage(path=XSDataString(imagePath), number=XSDataInteger(index)) listXSDataImage.append(xsDataImage) else: listXSDataImage = self.dataInput.image xsDataInputMXWaitFile = XSDataInputMXWaitFile() self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators() listPluginDistl = [] listPluginDozor = [] listOfImagesInBatch = [] listOfAllBatches = [] indexBatch = 0 listH5FilePath = [] # Process data in batches for xsDataImage in listXSDataImage: listOfImagesInBatch.append(xsDataImage.copy()) if len(listOfImagesInBatch) == batchSize: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] if len(listOfImagesInBatch) > 0: listOfAllBatches.append(listOfImagesInBatch) listOfImagesInBatch = [] # Loop over batches for listOfImagesInBatch in listOfAllBatches: # First wait for images for image in listOfImagesInBatch: strPathToImage = image.path.value # If Eiger, just wait for the h5 file if strPathToImage.endswith(".h5"): h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage, batchSize=batchSize, isFastMesh=isFastMesh) # print(h5FilePath) # print(hdf5ImageNumber) if not h5DataFilePath in listH5FilePath: self.screen("ID30a3 Eiger data, waiting for master and data files...") listH5FilePath.append(h5DataFilePath) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Waiting for file {0}".format(h5DataFilePath)) self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() # hdf5FilePath = strPathToImage.replace(".cbf", ".h5") time.sleep(1) if not os.path.exists(h5DataFilePath): strError = "Time-out while waiting for image %s" % h5DataFilePath self.error(strError) self.addErrorMessage(strError) self.setFailure() else: if not os.path.exists(strPathToImage): # self.screen("Waiting for file {0}".format(strPathToImage)) self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName) xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage)) xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize)) xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut)) self.screen("Wait file timeOut set to %.0f s" % self.fMXWaitFileTimeOut) self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile) self.edPluginMXWaitFile.executeSynchronous() if not os.path.exists(strPathToImage): strError = "Time-out while waiting for image %s" % strPathToImage self.error(strError) self.addErrorMessage(strError) self.setFailure() if not self.isFailure(): strPathToFirstImage = listOfImagesInBatch[0].path.value if strPathToImage.endswith(".h5"): indexLoop = 1 continueLoop = True while continueLoop: directory = os.path.dirname(strPathToFirstImage) firstImage = EDUtilsImage.getImageNumber(listOfImagesInBatch[0].path.value) lastImage = EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value) xsDataInputH5ToCBF = XSDataInputH5ToCBF() xsDataInputH5ToCBF.hdf5File = XSDataFile(listOfImagesInBatch[0].path) xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1) xsDataInputH5ToCBF.startImageNumber = XSDataInteger(firstImage) xsDataInputH5ToCBF.endImageNumber = XSDataInteger(lastImage) xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile(XSDataString(directory)) edPluginH5ToCBF = self.loadPlugin("EDPluginH5ToCBFv1_1") edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF edPluginH5ToCBF.execute() edPluginH5ToCBF.synchronize() outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate if outputCBFFileTemplate is not None: lastCbfFile = outputCBFFileTemplate.path.value.replace("######", "{0:06d}".format(EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value))) strPathToImage = os.path.join(directory, lastCbfFile) # print(cbfFile.path.value) if os.path.exists(strPathToImage): # Rename all images for image in listOfImagesInBatch: image.path.value = image.path.value.replace(".h5", ".cbf") imageNumber = EDUtilsImage.getImageNumber(image.path.value) oldPath = os.path.join(directory, outputCBFFileTemplate.path.value.replace("######", "{0:06d}".format(imageNumber))) newPath = os.path.join(directory, outputCBFFileTemplate.path.value.replace("######", "{0:04d}".format(imageNumber))) os.rename(oldPath, newPath) lastCbfFile = outputCBFFileTemplate.path.value.replace("######", "{0:04d}".format(EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value))) strPathToImage = os.path.join(directory, lastCbfFile) self.screen("Image has been converted to CBF file: {0}".format(strPathToImage)) continueLoop = False # print(continueLoop) if continueLoop: self.screen("Still waiting for converting to CBF file: {0}".format(strPathToImage)) indexLoop += 1 time.sleep(5) if indexLoop > 10: continueLoop = False for image in listOfImagesInBatch: strPathToImage = image.path.value # Check if we should run distl.signalStrength xsDataImageNew = XSDataImage(XSDataString(strPathToImage)) xsDataImageNew.number = XSDataInteger(EDUtilsImage.getImageNumber(image.path.value)) edPluginPluginExecImageQualityIndicator = None if bDoDistlSignalStrength: if self.bUseThinClient: strPluginName = self.strPluginNameThinClient else: strPluginName = self.strPluginName edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName) self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator) xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength() xsDataInputDistlSignalStrength.setReferenceImage(xsDataImageNew) edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength) edPluginPluginExecImageQualityIndicator.execute() listPluginDistl.append((xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator)) edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor, "ControlDozor_{0}".format(os.path.splitext(os.path.basename(strPathToFirstImage))[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listOfImagesInBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(len(listOfImagesInBatch)) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.execute() listPluginDozor.append((edPluginControlDozor, list(listOfImagesInBatch))) if not self.isFailure(): listIndexing = [] # Synchronize all image quality indicator plugins and upload to ISPyB xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators() for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl: xsDataImageQualityIndicators = XSDataImageQualityIndicators() xsDataImageQualityIndicators.image = xsDataImage.copy() if edPluginPluginExecImageQualityIndicator is not None: edPluginPluginExecImageQualityIndicator.synchronize() if edPluginPluginExecImageQualityIndicator.dataOutput is not None: if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None: xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\ edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal()) self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators) for (edPluginControlDozor, listBatch) in listPluginDozor: edPluginControlDozor.synchronize() # Check that we got at least one result if len(edPluginControlDozor.dataOutput.imageDozor) == 0: # Run the dozor plugin again, this time synchronously firstImage = os.path.basename(listBatch[0].path.value) lastImage = os.path.basename(listBatch[-1].path.value) self.screen("No dozor results! Re-executing Dozor for images {0} to {1}".format(firstImage, lastImage)) time.sleep(5) edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor, "ControlDozor_reexecution_{0}".format(os.path.splitext(firstImage)[0])) xsDataInputControlDozor = XSDataInputControlDozor() for image in listBatch: xsDataInputControlDozor.addImage(XSDataFile(image.path)) xsDataInputControlDozor.batchSize = XSDataInteger(batchSize) edPluginControlDozor.dataInput = xsDataInputControlDozor edPluginControlDozor.executeSynchronous() for imageDozor in edPluginControlDozor.dataOutput.imageDozor: for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value: xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile if imageDozor.spotFile is not None: if os.path.exists(imageDozor.spotFile.path.value): numpyArray = numpy.loadtxt(imageDozor.spotFile.path.value, skiprows=3) xsDataImageQualityIndicators.dozorSpotList = XSDataString(base64.b64encode(numpyArray.tostring())) xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[0])) if len(numpyArray.shape) > 1: xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[1])) xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution if self.xsDataResultControlImageQualityIndicators.inputDozor is None: if edPluginControlDozor.dataOutput.inputDozor is not None: self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString( edPluginControlDozor.dataOutput.inputDozor.marshal()) if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: xsDataISPyBImageQualityIndicators = \ XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal()) xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators) # print xsDataInputStoreListOfImageQualityIndicators.marshal() if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value: self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName) self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators self.edPluginISPyB.execute() # if bDoIndexing: # Find the 5 most intensive images (TIS): listImage = [] # Check that we have dozor_score from all images: has_dozor_score = True for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators: if imageQualityIndicators.dozor_score is None: has_dozor_score = False if has_dozor_score: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value) else: listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators, key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value) for xsDataResultControlImageQualityIndicator in listSorted[-5:]: if xsDataResultControlImageQualityIndicator.dozor_score.value > 1: xsDataInputReadImageHeader = XSDataInputReadImageHeader() xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path) self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName) self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader self.edPluginReadImageHeader.executeSynchronous() xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput if xsDataResultReadImageHeader is not None: edPluginLabelitIndexing = self.loadPlugin(self.strIndexingLabelitPluginName) xsDataInputLabelitIndexing = XSDataInputLabelitIndexing() xsDataInputLabelitIndexing.image.append(XSDataImage(xsDataResultControlImageQualityIndicator.image.path)) edPluginLabelitIndexing.setDataInput(xsDataInputLabelitIndexing) self.listPluginLabelit.append([edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator]) edPluginLabelitIndexing.execute() for tupleLabelit in self.listPluginLabelit: edPluginLabelitIndexing = tupleLabelit[0] xsDataResultControlImageQualityIndicator = tupleLabelit[1] edPluginLabelitIndexing.synchronize() if not edPluginLabelitIndexing.isFailure() and edPluginLabelitIndexing.dataOutput is not None: xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput() xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(xsDataResultLabelitIndexing) selectedSolution = xsDataIndexingResult.selectedSolution if selectedSolution is not None: xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution def finallyProcess(self, _edPlugin=None): EDPluginControl.finallyProcess(self, _edPlugin) if self.edPluginISPyB is not None: # Synchronize ISPyB plugin self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.finallyProcess") self.edPluginISPyB.synchronize() listId = [] for xsDataInteger in self.edPluginISPyB.dataOutput.imageQualityIndicatorsId: listId.append(xsDataInteger.value) self.DEBUG("ISPyB imageQualityIndicatorIds = %r" % listId) self.setDataOutput(self.xsDataResultControlImageQualityIndicators) def generateExecutiveSummary(self, _edPlugin=None): self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.generateExecutiveSummary") self.addErrorWarningMessagesToExecutiveSummary("Image quality indicator plugin execution failure! Error messages: ") self.addExecutiveSummaryLine("Summary of image quality indicators with %s :" % self.strPluginName) for edPluginPluginExecImageQualityIndicator in self.listPluginExecImageQualityIndicator: self.addExecutiveSummaryLine("") if edPluginPluginExecImageQualityIndicator is not None: self.appendExecutiveSummary(edPluginPluginExecImageQualityIndicator, "Distl.signal_strength : ", _bAddSeparator=False) def getH5FilePath(self, filePath, batchSize=1, isFastMesh=False): imageNumber = EDUtilsImage.getImageNumber(filePath) prefix = EDUtilsImage.getPrefix(filePath) if isFastMesh: h5ImageNumber = int((imageNumber - 1) / 100) + 1 h5FileNumber = 1 else: h5ImageNumber = 1 h5FileNumber = int((imageNumber - 1) / batchSize) * batchSize + 1 h5MasterFileName = "{prefix}_{h5FileNumber}_master.h5".format(prefix=prefix, h5FileNumber=h5FileNumber) h5MasterFilePath = os.path.join(os.path.dirname(filePath), h5MasterFileName) h5DataFileName = "{prefix}_{h5FileNumber}_data_{h5ImageNumber:06d}.h5".format(prefix=prefix, h5FileNumber=h5FileNumber, h5ImageNumber=h5ImageNumber) h5DataFilePath = os.path.join(os.path.dirname(filePath), h5DataFileName) return h5MasterFilePath, h5DataFilePath, h5FileNumber