def preProcess(self, _edObject=None):
     EDPluginControl.preProcess(self)
     self.DEBUG("EDPluginControlIndexingIndicatorsv1_1.preProcess")
     # Load and prepare the execution plugin
     self.edPluginIndexingLabelit = self.loadPlugin(self.strPluginIndexingLabelit)
     self.edPluginIndexingLabelit.setUseWarningInsteadOfError(True)
     xsDataIndexingInput = XSDataIndexingInput()
     xsDataIndexingInput.setDataCollection(self.getDataInput("dataCollection")[0])
     if self.hasDataInput("crystal"):
         xsDataIndexingInput.setCrystal(self.getDataInput("crystal")[0])
     if self.hasDataInput("refinedExperimentalCondition"):
         self.xsDataExperimentalCondition = self.getDataInput("refinedExperimentalCondition")[0]
     else:
         self.xsDataExperimentalCondition = self.getDataInput("dataCollection")[0].getSubWedge()[0].getExperimentalCondition()
     xsDataIndexingInput.setExperimentalCondition(self.xsDataExperimentalCondition)
     from EDHandlerXSDataPhenixv1_1 import EDHandlerXSDataPhenixv1_1
     xsDataInputLabelitIndexing = EDHandlerXSDataPhenixv1_1.generateXSDataInputLabelitIndexing(xsDataIndexingInput)
     self.edPluginIndexingLabelit.setDataInput(xsDataInputLabelitIndexing)
     #
     if self.bDoLabelitIndexing:
         if (self.getControlledPluginName("indicatorsPlugin") is not None):
             self.strControlledIndicatorsPluginName = self.getControlledPluginName("indicatorsPlugin")
     self.edPluginControlIndicators = self.loadPlugin(self.strControlledIndicatorsPluginName)
     # Extract the images from the data collections
     xsDataSubWedgeList = self.getDataInput("dataCollection")[0].getSubWedge()
     xsDataInputControlImageQualityIndicators = XSDataInputControlImageQualityIndicators()
     for xsDataSubWedge in xsDataSubWedgeList:
         xsDataImageList = xsDataSubWedge.getImage()
         for xsDataImage in xsDataImageList:
             xsDataInputControlImageQualityIndicators.addImage(xsDataImage)
     self.edPluginControlIndicators.setDataInput(xsDataInputControlImageQualityIndicators)
 def loadPluginIndexingInputData(self):
     self.verboseDebug(
         "EDPluginControlIndexingLabelitv10.loadPluginIndexingInputData...")
     xsDataIndexingInput = self.getDataInput()
     from EDHandlerXSDataPhenixv1_1 import EDHandlerXSDataPhenixv1_1
     self.getPluginIndexing(
     ).dataInput = EDHandlerXSDataPhenixv1_1.generateXSDataInputLabelitIndexing(
         xsDataIndexingInput)
 def getDataIndexingResult(self, _edPlugin):
     """
     This method retrieves the indexing results from a MOSFLM indexing plugin.
     """
     self.DEBUG("EDPluginControlIndexingLabelitv10.getDataIndexingResultFromMOSFLM")
     xsDataResultLabelitIndexing = _edPlugin.dataOutput
     from EDHandlerXSDataPhenixv1_1 import EDHandlerXSDataPhenixv1_1
     xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(xsDataResultLabelitIndexing,
                                                                                    self.getExperimentalCondition())
     xsDataIndexingResult.setLabelitIndexing(XSDataBoolean(True))
     return xsDataIndexingResult
 def getDataIndexingResult(self, _edPlugin):
     """
     This method retrieves the indexing results from a MOSFLM indexing plugin.
     """
     self.DEBUG(
         "EDPluginControlIndexingLabelitv10.getDataIndexingResultFromMOSFLM"
     )
     xsDataResultLabelitIndexing = _edPlugin.dataOutput
     from EDHandlerXSDataPhenixv1_1 import EDHandlerXSDataPhenixv1_1
     xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(
         xsDataResultLabelitIndexing, self.getExperimentalCondition())
     xsDataIndexingResult.setLabelitIndexing(XSDataBoolean(True))
     return xsDataIndexingResult
    def doSuccessLabelitIndexing(self, _edPlugin=None):
        self.DEBUG("EDPluginControlIndexingIndicatorsv1_1.doSuccessLabelitIndexing")
        self.synchronizeOn()
        xsDataResultLabelitIndexing = _edPlugin.dataOutput
        from EDHandlerXSDataPhenixv1_1 import EDHandlerXSDataPhenixv1_1
        xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(xsDataResultLabelitIndexing,
                                                                                       self.xsDataExperimentalCondition)
        xsDataCollection = self.getDataInput("dataCollection")[0]
        xsDataListImage = self.generateImageList(xsDataCollection)
        xsDataIndexingResult.setImage(xsDataListImage)
        xsDataIndexingResult.setLabelitIndexing(XSDataBoolean(False))
        self.setDataOutput(xsDataIndexingResult, "indexingResult")
#        self.generateExecutiveSummaryLabelit(_edPlugin)
        self.addExecutiveSummarySeparator()
        self.addExecutiveSummaryLine("Summary of indexing with %s :" % self.strControlledIndicatorsPluginName)
        self.addExecutiveSummaryLine("")
        self.appendExecutiveSummary(self.edPluginIndexingLabelit, "Labelit : ", _bAddSeparator=False)
        self.generateIndexingShortSummary(xsDataIndexingResult)
        self.synchronizeOff()
Exemple #6
0
 def preProcess(self, _edObject=None):
     EDPluginControl.preProcess(self)
     self.DEBUG("EDPluginControlIndexingIndicatorsv1_1.preProcess")
     # Load and prepare the execution plugin
     self.edPluginIndexingLabelit = self.loadPlugin(
         self.strPluginIndexingLabelit)
     self.edPluginIndexingLabelit.setUseWarningInsteadOfError(True)
     xsDataIndexingInput = XSDataIndexingInput()
     xsDataIndexingInput.setDataCollection(
         self.getDataInput("dataCollection")[0])
     if self.hasDataInput("crystal"):
         xsDataIndexingInput.setCrystal(self.getDataInput("crystal")[0])
     if self.hasDataInput("refinedExperimentalCondition"):
         self.xsDataExperimentalCondition = self.getDataInput(
             "refinedExperimentalCondition")[0]
     else:
         self.xsDataExperimentalCondition = self.getDataInput(
             "dataCollection")[0].getSubWedge()[0].getExperimentalCondition(
             )
     xsDataIndexingInput.setExperimentalCondition(
         self.xsDataExperimentalCondition)
     from EDHandlerXSDataPhenixv1_1 import EDHandlerXSDataPhenixv1_1
     xsDataInputLabelitIndexing = EDHandlerXSDataPhenixv1_1.generateXSDataInputLabelitIndexing(
         xsDataIndexingInput)
     self.edPluginIndexingLabelit.setDataInput(xsDataInputLabelitIndexing)
     #
     if self.bDoLabelitIndexing:
         if (self.getControlledPluginName("indicatorsPlugin") is not None):
             self.strControlledIndicatorsPluginName = self.getControlledPluginName(
                 "indicatorsPlugin")
     self.edPluginControlIndicators = self.loadPlugin(
         self.strControlledIndicatorsPluginName)
     # Extract the images from the data collections
     xsDataSubWedgeList = self.getDataInput(
         "dataCollection")[0].getSubWedge()
     xsDataInputControlImageQualityIndicators = XSDataInputControlImageQualityIndicators(
     )
     for xsDataSubWedge in xsDataSubWedgeList:
         xsDataImageList = xsDataSubWedge.getImage()
         for xsDataImage in xsDataImageList:
             xsDataInputControlImageQualityIndicators.addImage(xsDataImage)
     self.edPluginControlIndicators.setDataInput(
         xsDataInputControlImageQualityIndicators)
Exemple #7
0
 def doSuccessLabelitIndexing(self, _edPlugin=None):
     self.DEBUG(
         "EDPluginControlIndexingIndicatorsv1_1.doSuccessLabelitIndexing")
     self.synchronizeOn()
     xsDataResultLabelitIndexing = _edPlugin.dataOutput
     from EDHandlerXSDataPhenixv1_1 import EDHandlerXSDataPhenixv1_1
     xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(
         xsDataResultLabelitIndexing, self.xsDataExperimentalCondition)
     xsDataCollection = self.getDataInput("dataCollection")[0]
     xsDataListImage = self.generateImageList(xsDataCollection)
     xsDataIndexingResult.setImage(xsDataListImage)
     xsDataIndexingResult.setLabelitIndexing(XSDataBoolean(False))
     self.setDataOutput(xsDataIndexingResult, "indexingResult")
     #        self.generateExecutiveSummaryLabelit(_edPlugin)
     self.addExecutiveSummarySeparator()
     self.addExecutiveSummaryLine("Summary of indexing with %s :" %
                                  self.strControlledIndicatorsPluginName)
     self.addExecutiveSummaryLine("")
     self.appendExecutiveSummary(self.edPluginIndexingLabelit,
                                 "Labelit : ",
                                 _bAddSeparator=False)
     self.generateIndexingShortSummary(xsDataIndexingResult)
     self.synchronizeOff()
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
        EDUtilsParallel.initializeNbThread()
        # Check batch size
        if self.dataInput.batchSize is None:
            batchSize = 1
        else:
            batchSize = self.dataInput.batchSize.value
        # Check if we should do distlSignalStrength:
        bDoDistlSignalStrength = True
        if self.dataInput.doDistlSignalStrength is not None:
            if not self.dataInput.doDistlSignalStrength.value:
                bDoDistlSignalStrength = False
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                 bDoIndexing = True
        # Loop through all the incoming reference images
        listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators()
        listPluginDistl = []
        listPluginDozor = []
        listBatch = []
        indexBatch = 0
        for xsDataImage in listXSDataImage:
            self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
            xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
            xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
            xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
            self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
            self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
            self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(xsDataImage.path.value):
                self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
                xsDataInputMXWaitFile.setSize(XSDataInteger(5000000))
                xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
                self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(xsDataImage.path.value):
                strError = "Time-out while waiting for image %s" % xsDataImage.path.value
                self.error(strError)
                self.addErrorMessage(strError)
                self.setFailure()
            else:
                # Check if we should run distl.signalStrength
                edPluginPluginExecImageQualityIndicator = None
                if bDoDistlSignalStrength:
                    if self.bUseThinClient:
                        strPluginName = self.strPluginNameThinClient
                    else:
                        strPluginName = self.strPluginName
                    edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName)
                    self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator)
                    xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength()
                    xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage)
                    edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength)
                    edPluginPluginExecImageQualityIndicator.execute()
                listPluginDistl.append((xsDataImage.copy(), edPluginPluginExecImageQualityIndicator))
                listBatch.append(xsDataImage.copy())
                if len(listBatch) == batchSize:
                    edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
                    xsDataInputControlDozor = XSDataInputControlDozor()
                    for image in listBatch:
                        xsDataInputControlDozor.addImage(XSDataFile(image.path))
                    xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
                    edPluginControlDozor.dataInput = xsDataInputControlDozor
                    edPluginControlDozor.execute()
                    listPluginDozor.append((edPluginControlDozor, listBatch))
                    listBatch = []
        if len(listBatch) > 0:
            # Process the remaining images...
            edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
            xsDataInputControlDozor = XSDataInputControlDozor()
            for image in listBatch:
                xsDataInputControlDozor.addImage(XSDataFile(image.path))
            xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
            edPluginControlDozor.dataInput = xsDataInputControlDozor
            edPluginControlDozor.execute()
            listPluginDozor.append([edPluginControlDozor, listBatch])
        listIndexing = []
        # Synchronize all image quality indicator plugins and upload to ISPyB
        xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators()

        for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl:
            xsDataImageQualityIndicators = XSDataImageQualityIndicators()
            xsDataImageQualityIndicators.image = xsDataImage.copy()
            if edPluginPluginExecImageQualityIndicator is not None:
                edPluginPluginExecImageQualityIndicator.synchronize()
                if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
                    xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
                            edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
            self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)

        for (edPluginControlDozor, listBatch) in listPluginDozor:
            edPluginControlDozor.synchronize()
            for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
                for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                    if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
                        xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
                        xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
                        xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
                        xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
                        if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
                            if edPluginControlDozor.dataOutput.inputDozor is not None:
                                self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString(
                                               edPluginControlDozor.dataOutput.inputDozor.marshal())
            if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
                xsDataISPyBImageQualityIndicators = \
                    XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
                xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators)
#        print xsDataInputStoreListOfImageQualityIndicators.marshal()
        if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
            self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
            self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
            self.edPluginISPyB.execute()
        #
        if bDoIndexing:
            # Find the 5 most intensive images (TIS):
            listImage = []
            # Check that we have dozor_score from all images:
            has_dozor_score = True
            for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                if imageQualityIndicators.dozor_score is None:
                    has_dozor_score = False
            if has_dozor_score:
                listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
                                    key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value)
            else:
                listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
                                    key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value)
            for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
                if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
                    xsDataInputReadImageHeader = XSDataInputReadImageHeader()
                    xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path)
                    self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName)
                    self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
                    self.edPluginReadImageHeader.executeSynchronous()
                    xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
                    if xsDataResultReadImageHeader is not None:
                        edPluginLabelitIndexing = self.loadPlugin(self.strIndexingLabelitPluginName)
                        xsDataInputLabelitIndexing = XSDataInputLabelitIndexing()
                        xsDataInputLabelitIndexing.image.append(XSDataImage(xsDataResultControlImageQualityIndicator.image.path))
                        edPluginLabelitIndexing.setDataInput(xsDataInputLabelitIndexing)
                        self.listPluginLabelit.append([edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator])
                        edPluginLabelitIndexing.execute()
            for tupleLabelit in self.listPluginLabelit:
                edPluginLabelitIndexing = tupleLabelit[0]
                xsDataResultControlImageQualityIndicator = tupleLabelit[1]
                edPluginLabelitIndexing.synchronize()
                if not edPluginLabelitIndexing.isFailure() and edPluginLabelitIndexing.dataOutput is not None:
                    xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput()
                    xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(xsDataResultLabelitIndexing)
                    selectedSolution = xsDataIndexingResult.selectedSolution
                    if selectedSolution is not None:
                        xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
        EDUtilsParallel.initializeNbThread()
        # Check batch size
        if self.dataInput.batchSize is None:
            batchSize = 1
        else:
            batchSize = self.dataInput.batchSize.value
        self.screen("Batch size: {0}".format(batchSize))
        # Check if we should do distlSignalStrength:
        bDoDistlSignalStrength = True
        if self.dataInput.doDistlSignalStrength is not None:
            if not self.dataInput.doDistlSignalStrength.value:
                bDoDistlSignalStrength = False
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                 bDoIndexing = True
        # Loop through all the incoming reference images
        listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators()
        listPluginDistl = []
        listPluginDozor = []
        listBatch = []
        indexBatch = 0
        listH5FilePath = []
        ispybDataCollection = None
        for xsDataImage in listXSDataImage:
            strPathToImage = xsDataImage.path.value
            # If Eiger, just wait for the h5 file
            if "id30a3" in strPathToImage:
                h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage, batchSize)
#                print(h5FilePath)
#                print(hdf5ImageNumber)
                if not h5DataFilePath in listH5FilePath:
                    self.screen("ID30a3 Eiger data, waiting for master and data files...")
                    listH5FilePath.append(h5DataFilePath)
                    self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                    xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath))
                    xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
                    xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                    self.screen("Waiting for file {0}".format(h5DataFilePath))
                    self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
                    self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                    self.edPluginMXWaitFile.executeSynchronous()
#                    hdf5FilePath = strPathToImage.replace(".cbf", ".h5")
                    ispybDataCollection = None
                    time.sleep(1)
                indexLoop = 1
                continueLoop = True
                while continueLoop:
                    xsDataInputControlH5ToCBF = XSDataInputControlH5ToCBF()
                    xsDataInputControlH5ToCBF.hdf5File = XSDataFile(XSDataString(strPathToImage))
                    imageNumber = EDUtilsImage.getImageNumber(strPathToImage)
                    xsDataInputControlH5ToCBF.imageNumber = XSDataInteger(imageNumber)
                    xsDataInputControlH5ToCBF.hdf5ImageNumber = XSDataInteger(hdf5ImageNumber)
                    xsDataInputControlH5ToCBF.ispybDataCollection = ispybDataCollection
                    edPluginControlH5ToCBF = self.loadPlugin(self.strPluginControlH5ToCBF, "ControlH5ToCBF_%04d_%02d" % (imageNumber, indexLoop))
                    edPluginControlH5ToCBF.dataInput = xsDataInputControlH5ToCBF
                    edPluginControlH5ToCBF.executeSynchronous()
                    cbfFile = edPluginControlH5ToCBF.dataOutput.outputCBFFile
#                    print(cbfFile)
#                    print(indexLoop)
                    if cbfFile is not None:
                        strPathToImage = cbfFile.path.value
#                        print(cbfFile.path.value)
                        if os.path.exists(strPathToImage):
                            self.screen("Image has been converted to CBF file: {0}".format(strPathToImage))
                            continueLoop = False
#                    print(continueLoop)
                    if continueLoop:
                        self.screen("Still waiting for converting to CBF file: {0}".format(strPathToImage))
                        indexLoop += 1
                        time.sleep(5)
                        if indexLoop > 10:
                            continueLoop = False

                ispybDataCollection = edPluginControlH5ToCBF.dataOutput.ispybDataCollection


            elif not os.path.exists(strPathToImage):
                self.screen("Waiting for file {0}".format(strPathToImage))
                self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage))
                xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
                xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
                self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(strPathToImage):
                strError = "Time-out while waiting for image %s" % strPathToImage
                self.error(strError)
                self.addErrorMessage(strError)
                self.setFailure()
            else:
                # Check if we should run distl.signalStrength
                xsDataImageNew = XSDataImage(XSDataString(strPathToImage))
                xsDataImageNew.number = xsDataImage.number
                xsDataImageNew.date = xsDataImage.date
                edPluginPluginExecImageQualityIndicator = None
                if bDoDistlSignalStrength:
                    if self.bUseThinClient:
                        strPluginName = self.strPluginNameThinClient
                    else:
                        strPluginName = self.strPluginName
                    edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName)
                    self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator)
                    xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength()
                    xsDataInputDistlSignalStrength.setReferenceImage(xsDataImageNew)
                    edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength)
                    edPluginPluginExecImageQualityIndicator.execute()
                listPluginDistl.append((xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator))
                listBatch.append(xsDataImageNew.copy())
                if len(listBatch) == batchSize:
                    edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
                    xsDataInputControlDozor = XSDataInputControlDozor()
                    for image in listBatch:
                        xsDataInputControlDozor.addImage(XSDataFile(image.path))
                    xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
                    edPluginControlDozor.dataInput = xsDataInputControlDozor
                    edPluginControlDozor.execute()
                    listPluginDozor.append((edPluginControlDozor, listBatch))
                    listBatch = []
        if len(listBatch) > 0:
            # Process the remaining images...
            edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
            xsDataInputControlDozor = XSDataInputControlDozor()
            for image in listBatch:
                xsDataInputControlDozor.addImage(XSDataFile(image.path))
            xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
            edPluginControlDozor.dataInput = xsDataInputControlDozor
            edPluginControlDozor.execute()
            listPluginDozor.append([edPluginControlDozor, listBatch])
        listIndexing = []
        # Synchronize all image quality indicator plugins and upload to ISPyB
        xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators()

        for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl:
            xsDataImageQualityIndicators = XSDataImageQualityIndicators()
            xsDataImageQualityIndicators.image = xsDataImage.copy()
            if edPluginPluginExecImageQualityIndicator is not None:
                edPluginPluginExecImageQualityIndicator.synchronize()
                if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
                    xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
                            edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
            self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)

        for (edPluginControlDozor, listBatch) in listPluginDozor:
            edPluginControlDozor.synchronize()
            for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
                for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                    if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
                        xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
                        xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
                        if imageDozor.spotFile is not None:
                            if os.path.exists(imageDozor.spotFile.path.value):
                                numpyArray = numpy.loadtxt(imageDozor.spotFile.path.value, skiprows=3)
                                xsDataImageQualityIndicators.dozorSpotList = XSDataString(base64.b64encode(numpyArray.tostring()))
                                xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[0]))
                                if len(numpyArray.shape) > 1:
                                    xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[1]))
                        xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
                        xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
                        if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
                            if edPluginControlDozor.dataOutput.inputDozor is not None:
                                self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString(
                                               edPluginControlDozor.dataOutput.inputDozor.marshal())
            if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
                xsDataISPyBImageQualityIndicators = \
                    XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
                xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators)
#        print xsDataInputStoreListOfImageQualityIndicators.marshal()
        if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
            self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
            self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
            self.edPluginISPyB.execute()
        #
        if bDoIndexing:
            # Find the 5 most intensive images (TIS):
            listImage = []
            # Check that we have dozor_score from all images:
            has_dozor_score = True
            for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                if imageQualityIndicators.dozor_score is None:
                    has_dozor_score = False
            if has_dozor_score:
                listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
                                    key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value)
            else:
                listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
                                    key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value)
            for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
                if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
                    xsDataInputReadImageHeader = XSDataInputReadImageHeader()
                    xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path)
                    self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName)
                    self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
                    self.edPluginReadImageHeader.executeSynchronous()
                    xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
                    if xsDataResultReadImageHeader is not None:
                        edPluginLabelitIndexing = self.loadPlugin(self.strIndexingLabelitPluginName)
                        xsDataInputLabelitIndexing = XSDataInputLabelitIndexing()
                        xsDataInputLabelitIndexing.image.append(XSDataImage(xsDataResultControlImageQualityIndicator.image.path))
                        edPluginLabelitIndexing.setDataInput(xsDataInputLabelitIndexing)
                        self.listPluginLabelit.append([edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator])
                        edPluginLabelitIndexing.execute()
            for tupleLabelit in self.listPluginLabelit:
                edPluginLabelitIndexing = tupleLabelit[0]
                xsDataResultControlImageQualityIndicator = tupleLabelit[1]
                edPluginLabelitIndexing.synchronize()
                if not edPluginLabelitIndexing.isFailure() and edPluginLabelitIndexing.dataOutput is not None:
                    xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput()
                    xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(xsDataResultLabelitIndexing)
                    selectedSolution = xsDataIndexingResult.selectedSolution
                    if selectedSolution is not None:
                        xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
        EDUtilsParallel.initializeNbThread()
        # Check batch size
        if self.dataInput.batchSize is None:
            batchSize = 1
        else:
            batchSize = self.dataInput.batchSize.value
        self.screen(
            "Image quality indicators batch size: {0}".format(batchSize))
        # Check if we should do distlSignalStrength:
        bDoDistlSignalStrength = True
        if self.dataInput.doDistlSignalStrength is not None:
            if not self.dataInput.doDistlSignalStrength.value:
                bDoDistlSignalStrength = False
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                bDoIndexing = True
        # Check if fast mesh (for HDF5)
        isFastMesh = False
        if self.dataInput.fastMesh:
            isFastMesh = self.dataInput.fastMesh.value
        # Loop through all the incoming reference images
        if len(self.dataInput.image) == 0:
            directory = self.dataInput.directory.path.value
            template = self.dataInput.template.value
            startNo = self.dataInput.startNo.value
            endNo = self.dataInput.endNo.value
            listXSDataImage = []
            for index in range(startNo, endNo + 1):
                imageName = template.replace("####", "{0:04d}".format(index))
                imagePath = os.path.join(directory, imageName)
                xsDataImage = XSDataImage(path=XSDataString(imagePath),
                                          number=XSDataInteger(index))
                listXSDataImage.append(xsDataImage)
        else:
            listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators(
        )
        listPluginDistl = []
        listPluginDozor = []
        listOfImagesInBatch = []
        listOfAllBatches = []
        indexBatch = 0
        listH5FilePath = []
        # Process data in batches
        for xsDataImage in listXSDataImage:
            listOfImagesInBatch.append(xsDataImage.copy())
            if len(listOfImagesInBatch) == batchSize:
                listOfAllBatches.append(listOfImagesInBatch)
                listOfImagesInBatch = []
        if len(listOfImagesInBatch) > 0:
            listOfAllBatches.append(listOfImagesInBatch)
            listOfImagesInBatch = []
        # Loop over batches
        for listOfImagesInBatch in listOfAllBatches:
            # First wait for images
            for image in listOfImagesInBatch:
                strPathToImage = image.path.value
                # If Eiger, just wait for the h5 file
                if strPathToImage.endswith(".h5"):
                    h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(
                        strPathToImage,
                        batchSize=batchSize,
                        isFastMesh=isFastMesh)
                    #                print(h5FilePath)
                    #                print(hdf5ImageNumber)
                    if not h5DataFilePath in listH5FilePath:
                        self.screen(
                            "ID30a3 Eiger data, waiting for master and data files..."
                        )
                        listH5FilePath.append(h5DataFilePath)
                        self.edPluginMXWaitFile = self.loadPlugin(
                            self.strPluginMXWaitFileName)
                        xsDataInputMXWaitFile.file = XSDataFile(
                            XSDataString(h5DataFilePath))
                        xsDataInputMXWaitFile.setSize(
                            XSDataInteger(self.minImageSize))
                        xsDataInputMXWaitFile.setTimeOut(
                            XSDataTime(self.fMXWaitFileTimeOut))
                        self.screen(
                            "Waiting for file {0}".format(h5DataFilePath))
                        self.DEBUG("Wait file timeOut set to %f" %
                                   self.fMXWaitFileTimeOut)
                        self.edPluginMXWaitFile.setDataInput(
                            xsDataInputMXWaitFile)
                        self.edPluginMXWaitFile.executeSynchronous()
                        #                    hdf5FilePath = strPathToImage.replace(".cbf", ".h5")
                        time.sleep(1)
                    if not os.path.exists(h5DataFilePath):
                        strError = "Time-out while waiting for image %s" % h5DataFilePath
                        self.error(strError)
                        self.addErrorMessage(strError)
                        self.setFailure()
                else:
                    if not os.path.exists(strPathToImage):
                        # self.screen("Waiting for file {0}".format(strPathToImage))
                        self.edPluginMXWaitFile = self.loadPlugin(
                            self.strPluginMXWaitFileName)
                        xsDataInputMXWaitFile.file = XSDataFile(
                            XSDataString(strPathToImage))
                        xsDataInputMXWaitFile.setSize(
                            XSDataInteger(self.minImageSize))
                        xsDataInputMXWaitFile.setTimeOut(
                            XSDataTime(self.fMXWaitFileTimeOut))
                        self.screen("Wait file timeOut set to %.0f s" %
                                    self.fMXWaitFileTimeOut)
                        self.edPluginMXWaitFile.setDataInput(
                            xsDataInputMXWaitFile)
                        self.edPluginMXWaitFile.executeSynchronous()
                    if not os.path.exists(strPathToImage):
                        strError = "Time-out while waiting for image %s" % strPathToImage
                        self.error(strError)
                        self.addErrorMessage(strError)
                        self.setFailure()
            if not self.isFailure():
                strPathToFirstImage = listOfImagesInBatch[0].path.value
                if strPathToImage.endswith(".h5"):
                    indexLoop = 1
                    continueLoop = True
                    while continueLoop:
                        directory = os.path.dirname(strPathToFirstImage)
                        firstImage = EDUtilsImage.getImageNumber(
                            listOfImagesInBatch[0].path.value)
                        lastImage = EDUtilsImage.getImageNumber(
                            listOfImagesInBatch[-1].path.value)
                        xsDataInputH5ToCBF = XSDataInputH5ToCBF()
                        xsDataInputH5ToCBF.hdf5File = XSDataFile(
                            listOfImagesInBatch[0].path)
                        xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1)
                        xsDataInputH5ToCBF.startImageNumber = XSDataInteger(
                            firstImage)
                        xsDataInputH5ToCBF.endImageNumber = XSDataInteger(
                            lastImage)
                        xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile(
                            XSDataString(directory))
                        edPluginH5ToCBF = self.loadPlugin(
                            "EDPluginH5ToCBFv1_1")
                        edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF
                        edPluginH5ToCBF.execute()
                        edPluginH5ToCBF.synchronize()
                        outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate
                        if outputCBFFileTemplate is not None:
                            lastCbfFile = outputCBFFileTemplate.path.value.replace(
                                "######", "{0:06d}".format(
                                    EDUtilsImage.getImageNumber(
                                        listOfImagesInBatch[-1].path.value)))
                            strPathToImage = os.path.join(
                                directory, lastCbfFile)
                            #                        print(cbfFile.path.value)
                            if os.path.exists(strPathToImage):
                                # Rename all images
                                for image in listOfImagesInBatch:
                                    image.path.value = image.path.value.replace(
                                        ".h5", ".cbf")
                                    imageNumber = EDUtilsImage.getImageNumber(
                                        image.path.value)
                                    oldPath = os.path.join(
                                        directory,
                                        outputCBFFileTemplate.path.value.
                                        replace("######",
                                                "{0:06d}".format(imageNumber)))
                                    newPath = os.path.join(
                                        directory,
                                        outputCBFFileTemplate.path.value.
                                        replace("######",
                                                "{0:04d}".format(imageNumber)))
                                    os.rename(oldPath, newPath)
                                lastCbfFile = outputCBFFileTemplate.path.value.replace(
                                    "######", "{0:04d}".format(
                                        EDUtilsImage.getImageNumber(
                                            listOfImagesInBatch[-1].path.value)
                                    ))
                                strPathToImage = os.path.join(
                                    directory, lastCbfFile)
                                self.screen(
                                    "Image has been converted to CBF file: {0}"
                                    .format(strPathToImage))
                                continueLoop = False
    #                    print(continueLoop)
                        if continueLoop:
                            self.screen(
                                "Still waiting for converting to CBF file: {0}"
                                .format(strPathToImage))
                            indexLoop += 1
                            time.sleep(5)
                            if indexLoop > 10:
                                continueLoop = False

                for image in listOfImagesInBatch:
                    strPathToImage = image.path.value
                    # Check if we should run distl.signalStrength
                    xsDataImageNew = XSDataImage(XSDataString(strPathToImage))
                    xsDataImageNew.number = XSDataInteger(
                        EDUtilsImage.getImageNumber(image.path.value))
                    edPluginPluginExecImageQualityIndicator = None
                    if bDoDistlSignalStrength:
                        if self.bUseThinClient:
                            strPluginName = self.strPluginNameThinClient
                        else:
                            strPluginName = self.strPluginName
                        edPluginPluginExecImageQualityIndicator = self.loadPlugin(
                            strPluginName)
                        self.listPluginExecImageQualityIndicator.append(
                            edPluginPluginExecImageQualityIndicator)
                        xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength(
                        )
                        xsDataInputDistlSignalStrength.setReferenceImage(
                            xsDataImageNew)
                        edPluginPluginExecImageQualityIndicator.setDataInput(
                            xsDataInputDistlSignalStrength)
                        edPluginPluginExecImageQualityIndicator.execute()
                    listPluginDistl.append(
                        (xsDataImageNew.copy(),
                         edPluginPluginExecImageQualityIndicator))

                edPluginControlDozor = self.loadPlugin(
                    self.strPluginNameControlDozor, "ControlDozor_{0}".format(
                        os.path.splitext(
                            os.path.basename(strPathToFirstImage))[0]))
                xsDataInputControlDozor = XSDataInputControlDozor()
                for image in listOfImagesInBatch:
                    xsDataInputControlDozor.addImage(XSDataFile(image.path))
                xsDataInputControlDozor.batchSize = XSDataInteger(
                    len(listOfImagesInBatch))
                edPluginControlDozor.dataInput = xsDataInputControlDozor
                edPluginControlDozor.execute()
                listPluginDozor.append(
                    (edPluginControlDozor, list(listOfImagesInBatch)))

        if not self.isFailure():
            listIndexing = []
            # Synchronize all image quality indicator plugins and upload to ISPyB
            xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators(
            )

            for (xsDataImage,
                 edPluginPluginExecImageQualityIndicator) in listPluginDistl:
                xsDataImageQualityIndicators = XSDataImageQualityIndicators()
                xsDataImageQualityIndicators.image = xsDataImage.copy()
                if edPluginPluginExecImageQualityIndicator is not None:
                    edPluginPluginExecImageQualityIndicator.synchronize()
                    if edPluginPluginExecImageQualityIndicator.dataOutput is not None:
                        if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
                            xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
                                    edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
                self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(
                    xsDataImageQualityIndicators)

            for (edPluginControlDozor, listBatch) in listPluginDozor:
                edPluginControlDozor.synchronize()
                # Check that we got at least one result
                if len(edPluginControlDozor.dataOutput.imageDozor) == 0:
                    # Run the dozor plugin again, this time synchronously
                    firstImage = os.path.basename(listBatch[0].path.value)
                    lastImage = os.path.basename(listBatch[-1].path.value)
                    self.screen(
                        "No dozor results! Re-executing Dozor for images {0} to {1}"
                        .format(firstImage, lastImage))
                    time.sleep(5)
                    edPluginControlDozor = self.loadPlugin(
                        self.strPluginNameControlDozor,
                        "ControlDozor_reexecution_{0}".format(
                            os.path.splitext(firstImage)[0]))
                    xsDataInputControlDozor = XSDataInputControlDozor()
                    for image in listBatch:
                        xsDataInputControlDozor.addImage(XSDataFile(
                            image.path))
                    xsDataInputControlDozor.batchSize = XSDataInteger(
                        batchSize)
                    edPluginControlDozor.dataInput = xsDataInputControlDozor
                    edPluginControlDozor.executeSynchronous()
                for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
                    for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                        if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
                            xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
                            xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
                            if imageDozor.spotFile is not None:
                                if os.path.exists(
                                        imageDozor.spotFile.path.value):
                                    numpyArray = numpy.loadtxt(
                                        imageDozor.spotFile.path.value,
                                        skiprows=3)
                                    xsDataImageQualityIndicators.dozorSpotList = XSDataString(
                                        base64.b64encode(
                                            numpyArray.tostring()))
                                    xsDataImageQualityIndicators.addDozorSpotListShape(
                                        XSDataInteger(numpyArray.shape[0]))
                                    if len(numpyArray.shape) > 1:
                                        xsDataImageQualityIndicators.addDozorSpotListShape(
                                            XSDataInteger(numpyArray.shape[1]))
                            xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
                            xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
                            xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution
                            if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
                                if edPluginControlDozor.dataOutput.inputDozor is not None:
                                    self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput(
                                    ).parseString(
                                        edPluginControlDozor.dataOutput.
                                        inputDozor.marshal())
                if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
                    xsDataISPyBImageQualityIndicators = \
                        XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
                    xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(
                        xsDataISPyBImageQualityIndicators)

    #        print xsDataInputStoreListOfImageQualityIndicators.marshal()
            if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
                self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
                self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
                self.edPluginISPyB.execute()
            #
            if bDoIndexing:
                # Find the 5 most intensive images (TIS):
                listImage = []
                # Check that we have dozor_score from all images:
                has_dozor_score = True
                for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                    if imageQualityIndicators.dozor_score is None:
                        has_dozor_score = False
                if has_dozor_score:
                    listSorted = sorted(
                        self.xsDataResultControlImageQualityIndicators.
                        imageQualityIndicators,
                        key=lambda imageQualityIndicators:
                        imageQualityIndicators.dozor_score.value)
                else:
                    listSorted = sorted(
                        self.xsDataResultControlImageQualityIndicators.
                        imageQualityIndicators,
                        key=lambda imageQualityIndicators:
                        imageQualityIndicators.totalIntegratedSignal.value)
                for xsDataResultControlImageQualityIndicator in listSorted[
                        -5:]:
                    if xsDataResultControlImageQualityIndicator.dozor_score.value > 1:
                        xsDataInputReadImageHeader = XSDataInputReadImageHeader(
                        )
                        xsDataInputReadImageHeader.image = XSDataFile(
                            xsDataResultControlImageQualityIndicator.image.path
                        )
                        self.edPluginReadImageHeader = self.loadPlugin(
                            self.strPluginReadImageHeaderName)
                        self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
                        self.edPluginReadImageHeader.executeSynchronous()
                        xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
                        if xsDataResultReadImageHeader is not None:
                            edPluginLabelitIndexing = self.loadPlugin(
                                self.strIndexingLabelitPluginName)
                            xsDataInputLabelitIndexing = XSDataInputLabelitIndexing(
                            )
                            xsDataInputLabelitIndexing.image.append(
                                XSDataImage(
                                    xsDataResultControlImageQualityIndicator.
                                    image.path))
                            edPluginLabelitIndexing.setDataInput(
                                xsDataInputLabelitIndexing)
                            self.listPluginLabelit.append([
                                edPluginLabelitIndexing,
                                xsDataResultControlImageQualityIndicator
                            ])
                            edPluginLabelitIndexing.execute()
                for tupleLabelit in self.listPluginLabelit:
                    edPluginLabelitIndexing = tupleLabelit[0]
                    xsDataResultControlImageQualityIndicator = tupleLabelit[1]
                    edPluginLabelitIndexing.synchronize()
                    if not edPluginLabelitIndexing.isFailure(
                    ) and edPluginLabelitIndexing.dataOutput is not None:
                        xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput(
                        )
                        xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(
                            xsDataResultLabelitIndexing)
                        selectedSolution = xsDataIndexingResult.selectedSolution
                        if selectedSolution is not None:
                            xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
 def loadPluginIndexingInputData(self):
     self.verboseDebug("EDPluginControlIndexingLabelitv10.loadPluginIndexingInputData...")
     xsDataIndexingInput = self.getDataInput()
     from EDHandlerXSDataPhenixv1_1 import EDHandlerXSDataPhenixv1_1
     self.getPluginIndexing().dataInput = EDHandlerXSDataPhenixv1_1.generateXSDataInputLabelitIndexing(xsDataIndexingInput)
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
        EDUtilsParallel.initializeNbThread()
        # Check batch size
        if self.dataInput.batchSize is None:
            batchSize = 1
        else:
            batchSize = self.dataInput.batchSize.value
        self.screen("Image quality indicators batch size: {0}".format(batchSize))
        # Check if we should do distlSignalStrength:
        bDoDistlSignalStrength = True
        if self.dataInput.doDistlSignalStrength is not None:
            if not self.dataInput.doDistlSignalStrength.value:
                bDoDistlSignalStrength = False
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                 bDoIndexing = True
        # Check if fast mesh (for HDF5)
        isFastMesh = False
        if self.dataInput.fastMesh:
            isFastMesh = self.dataInput.fastMesh.value
        # Loop through all the incoming reference images
        if len(self.dataInput.image) == 0:
            directory = self.dataInput.directory.path.value
            template = self.dataInput.template.value
            startNo = self.dataInput.startNo.value
            endNo = self.dataInput.endNo.value
            listXSDataImage = []
            for index in range(startNo, endNo + 1):
                imageName = template.replace("####", "{0:04d}".format(index))
                imagePath = os.path.join(directory, imageName)
                xsDataImage = XSDataImage(path=XSDataString(imagePath), number=XSDataInteger(index))
                listXSDataImage.append(xsDataImage)
        else:
            listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators()
        listPluginDistl = []
        listPluginDozor = []
        listOfImagesInBatch = []
        listOfAllBatches = []
        indexBatch = 0
        listH5FilePath = []
        # Process data in batches
        for xsDataImage in listXSDataImage:
            listOfImagesInBatch.append(xsDataImage.copy())
            if len(listOfImagesInBatch) == batchSize:
                listOfAllBatches.append(listOfImagesInBatch)
                listOfImagesInBatch = []
        if len(listOfImagesInBatch) > 0:
            listOfAllBatches.append(listOfImagesInBatch)
            listOfImagesInBatch = []
        # Loop over batches
        for listOfImagesInBatch in listOfAllBatches:
            # First wait for images
            for image in listOfImagesInBatch:
                strPathToImage = image.path.value
                # If Eiger, just wait for the h5 file
                if strPathToImage.endswith(".h5"):
                    h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage,
                                                                                           batchSize=batchSize,
                                                                                           isFastMesh=isFastMesh)
    #                print(h5FilePath)
    #                print(hdf5ImageNumber)
                    if not h5DataFilePath in listH5FilePath:
                        self.screen("ID30a3 Eiger data, waiting for master and data files...")
                        listH5FilePath.append(h5DataFilePath)
                        self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                        xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath))
                        xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
                        xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                        self.screen("Waiting for file {0}".format(h5DataFilePath))
                        self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
                        self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                        self.edPluginMXWaitFile.executeSynchronous()
    #                    hdf5FilePath = strPathToImage.replace(".cbf", ".h5")
                        time.sleep(1)
                    if not os.path.exists(h5DataFilePath):
                        strError = "Time-out while waiting for image %s" % h5DataFilePath
                        self.error(strError)
                        self.addErrorMessage(strError)
                        self.setFailure()
                else:
                    if not os.path.exists(strPathToImage):
                        # self.screen("Waiting for file {0}".format(strPathToImage))
                        self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                        xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage))
                        xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
                        xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                        self.screen("Wait file timeOut set to %.0f s" % self.fMXWaitFileTimeOut)
                        self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                        self.edPluginMXWaitFile.executeSynchronous()
                    if not os.path.exists(strPathToImage):
                        strError = "Time-out while waiting for image %s" % strPathToImage
                        self.error(strError)
                        self.addErrorMessage(strError)
                        self.setFailure()
            if not self.isFailure():
                strPathToFirstImage = listOfImagesInBatch[0].path.value
                if strPathToImage.endswith(".h5"):
                    indexLoop = 1
                    continueLoop = True
                    while continueLoop:
                        directory = os.path.dirname(strPathToFirstImage)
                        firstImage = EDUtilsImage.getImageNumber(listOfImagesInBatch[0].path.value)
                        lastImage = EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value)
                        xsDataInputH5ToCBF = XSDataInputH5ToCBF()
                        xsDataInputH5ToCBF.hdf5File = XSDataFile(listOfImagesInBatch[0].path)
                        xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1)
                        xsDataInputH5ToCBF.startImageNumber = XSDataInteger(firstImage)
                        xsDataInputH5ToCBF.endImageNumber = XSDataInteger(lastImage)
                        xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile(XSDataString(directory))
                        edPluginH5ToCBF = self.loadPlugin("EDPluginH5ToCBFv1_1")
                        edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF
                        edPluginH5ToCBF.execute()
                        edPluginH5ToCBF.synchronize()
                        outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate
                        if outputCBFFileTemplate is not None:
                            lastCbfFile = outputCBFFileTemplate.path.value.replace("######", "{0:06d}".format(EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value)))
                            strPathToImage = os.path.join(directory, lastCbfFile)
    #                        print(cbfFile.path.value)
                            if os.path.exists(strPathToImage):
                                # Rename all images
                                for image in listOfImagesInBatch:
                                    image.path.value = image.path.value.replace(".h5", ".cbf")
                                    imageNumber = EDUtilsImage.getImageNumber(image.path.value)
                                    oldPath = os.path.join(directory, outputCBFFileTemplate.path.value.replace("######", "{0:06d}".format(imageNumber)))
                                    newPath = os.path.join(directory, outputCBFFileTemplate.path.value.replace("######", "{0:04d}".format(imageNumber)))
                                    os.rename(oldPath, newPath)
                                lastCbfFile = outputCBFFileTemplate.path.value.replace("######", "{0:04d}".format(EDUtilsImage.getImageNumber(listOfImagesInBatch[-1].path.value)))
                                strPathToImage = os.path.join(directory, lastCbfFile)
                                self.screen("Image has been converted to CBF file: {0}".format(strPathToImage))
                                continueLoop = False
    #                    print(continueLoop)
                        if continueLoop:
                            self.screen("Still waiting for converting to CBF file: {0}".format(strPathToImage))
                            indexLoop += 1
                            time.sleep(5)
                            if indexLoop > 10:
                                continueLoop = False

                for image in listOfImagesInBatch:
                    strPathToImage = image.path.value
                    # Check if we should run distl.signalStrength
                    xsDataImageNew = XSDataImage(XSDataString(strPathToImage))
                    xsDataImageNew.number = XSDataInteger(EDUtilsImage.getImageNumber(image.path.value))
                    edPluginPluginExecImageQualityIndicator = None
                    if bDoDistlSignalStrength:
                        if self.bUseThinClient:
                            strPluginName = self.strPluginNameThinClient
                        else:
                            strPluginName = self.strPluginName
                        edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName)
                        self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator)
                        xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength()
                        xsDataInputDistlSignalStrength.setReferenceImage(xsDataImageNew)
                        edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength)
                        edPluginPluginExecImageQualityIndicator.execute()
                    listPluginDistl.append((xsDataImageNew.copy(), edPluginPluginExecImageQualityIndicator))

                edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor,
                                                       "ControlDozor_{0}".format(os.path.splitext(os.path.basename(strPathToFirstImage))[0]))
                xsDataInputControlDozor = XSDataInputControlDozor()
                for image in listOfImagesInBatch:
                    xsDataInputControlDozor.addImage(XSDataFile(image.path))
                xsDataInputControlDozor.batchSize = XSDataInteger(len(listOfImagesInBatch))
                edPluginControlDozor.dataInput = xsDataInputControlDozor
                edPluginControlDozor.execute()
                listPluginDozor.append((edPluginControlDozor, list(listOfImagesInBatch)))

        if not self.isFailure():
            listIndexing = []
            # Synchronize all image quality indicator plugins and upload to ISPyB
            xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators()

            for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl:
                xsDataImageQualityIndicators = XSDataImageQualityIndicators()
                xsDataImageQualityIndicators.image = xsDataImage.copy()
                if edPluginPluginExecImageQualityIndicator is not None:
                    edPluginPluginExecImageQualityIndicator.synchronize()
                    if edPluginPluginExecImageQualityIndicator.dataOutput is not None:
                        if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
                            xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
                                    edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
                self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)

            for (edPluginControlDozor, listBatch) in listPluginDozor:
                edPluginControlDozor.synchronize()
                # Check that we got at least one result
                if len(edPluginControlDozor.dataOutput.imageDozor) == 0:
                    # Run the dozor plugin again, this time synchronously
                    firstImage = os.path.basename(listBatch[0].path.value)
                    lastImage = os.path.basename(listBatch[-1].path.value)
                    self.screen("No dozor results! Re-executing Dozor for images {0} to {1}".format(firstImage, lastImage))
                    time.sleep(5)
                    edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor, "ControlDozor_reexecution_{0}".format(os.path.splitext(firstImage)[0]))
                    xsDataInputControlDozor = XSDataInputControlDozor()
                    for image in listBatch:
                        xsDataInputControlDozor.addImage(XSDataFile(image.path))
                    xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
                    edPluginControlDozor.dataInput = xsDataInputControlDozor
                    edPluginControlDozor.executeSynchronous()
                for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
                    for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                        if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
                            xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
                            xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
                            if imageDozor.spotFile is not None:
                                if os.path.exists(imageDozor.spotFile.path.value):
                                    numpyArray = numpy.loadtxt(imageDozor.spotFile.path.value, skiprows=3)
                                    xsDataImageQualityIndicators.dozorSpotList = XSDataString(base64.b64encode(numpyArray.tostring()))
                                    xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[0]))
                                    if len(numpyArray.shape) > 1:
                                        xsDataImageQualityIndicators.addDozorSpotListShape(XSDataInteger(numpyArray.shape[1]))
                            xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
                            xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
                            xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution
                            if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
                                if edPluginControlDozor.dataOutput.inputDozor is not None:
                                    self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput().parseString(
                                                   edPluginControlDozor.dataOutput.inputDozor.marshal())
                if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
                    xsDataISPyBImageQualityIndicators = \
                        XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
                    xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(xsDataISPyBImageQualityIndicators)
    #        print xsDataInputStoreListOfImageQualityIndicators.marshal()
            if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
                self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
                self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
                self.edPluginISPyB.execute()
            #
            if bDoIndexing:
                # Find the 5 most intensive images (TIS):
                listImage = []
                # Check that we have dozor_score from all images:
                has_dozor_score = True
                for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
                    if imageQualityIndicators.dozor_score is None:
                        has_dozor_score = False
                if has_dozor_score:
                    listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
                                        key=lambda imageQualityIndicators: imageQualityIndicators.dozor_score.value)
                else:
                    listSorted = sorted(self.xsDataResultControlImageQualityIndicators.imageQualityIndicators,
                                        key=lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value)
                for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
                    if xsDataResultControlImageQualityIndicator.dozor_score.value > 1:
                        xsDataInputReadImageHeader = XSDataInputReadImageHeader()
                        xsDataInputReadImageHeader.image = XSDataFile(xsDataResultControlImageQualityIndicator.image.path)
                        self.edPluginReadImageHeader = self.loadPlugin(self.strPluginReadImageHeaderName)
                        self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
                        self.edPluginReadImageHeader.executeSynchronous()
                        xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
                        if xsDataResultReadImageHeader is not None:
                            edPluginLabelitIndexing = self.loadPlugin(self.strIndexingLabelitPluginName)
                            xsDataInputLabelitIndexing = XSDataInputLabelitIndexing()
                            xsDataInputLabelitIndexing.image.append(XSDataImage(xsDataResultControlImageQualityIndicator.image.path))
                            edPluginLabelitIndexing.setDataInput(xsDataInputLabelitIndexing)
                            self.listPluginLabelit.append([edPluginLabelitIndexing, xsDataResultControlImageQualityIndicator])
                            edPluginLabelitIndexing.execute()
                for tupleLabelit in self.listPluginLabelit:
                    edPluginLabelitIndexing = tupleLabelit[0]
                    xsDataResultControlImageQualityIndicator = tupleLabelit[1]
                    edPluginLabelitIndexing.synchronize()
                    if not edPluginLabelitIndexing.isFailure() and edPluginLabelitIndexing.dataOutput is not None:
                        xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput()
                        xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(xsDataResultLabelitIndexing)
                        selectedSolution = xsDataIndexingResult.selectedSolution
                        if selectedSolution is not None:
                            xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution