Пример #1
0
 def getDataCollectionGroupId(self, _dataCollectionId):
     self.DEBUG("EDPluginControlInterfacev1_2.getDataCollectionGroupId")
     self.DEBUG("dataCollectionId = {0}".format(_dataCollectionId))
     dataCollectionGroupId = None
     xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
     xsDataInputRetrieveDataCollection.dataCollectionId = XSDataInteger(_dataCollectionId)
     edPluginISPyBRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
     edPluginISPyBRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
     edPluginISPyBRetrieveDataCollection.executeSynchronous()
     xsDataResultRetrieveDataCollection = edPluginISPyBRetrieveDataCollection.dataOutput
     if xsDataResultRetrieveDataCollection is not None:
         dataCollection = xsDataResultRetrieveDataCollection.dataCollection
         if dataCollection is not None:
             dataCollectionGroupId = dataCollection.dataCollectionGroupId
     self.DEBUG("dataCollectionGroupId = {0}".format(dataCollectionGroupId))
     return dataCollectionGroupId
 def getDataCollectionGroupId(self, _dataCollectionId):
     self.DEBUG("EDPluginControlInterfacev1_2.getDataCollectionGroupId")
     self.DEBUG("dataCollectionId = {0}".format(_dataCollectionId))
     dataCollectionGroupId = None
     xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
     xsDataInputRetrieveDataCollection.dataCollectionId = XSDataInteger(_dataCollectionId)
     edPluginISPyBRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
     edPluginISPyBRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
     edPluginISPyBRetrieveDataCollection.executeSynchronous()
     xsDataResultRetrieveDataCollection = edPluginISPyBRetrieveDataCollection.dataOutput
     if xsDataResultRetrieveDataCollection is not None:
         dataCollection = xsDataResultRetrieveDataCollection.dataCollection
         if dataCollection is not None:
             dataCollectionGroupId = dataCollection.dataCollectionGroupId
     self.DEBUG("dataCollectionGroupId = {0}".format(dataCollectionGroupId))
     return dataCollectionGroupId
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        pathToStartImage = None
        pathToEndImage = None
        userName = os.environ["USER"]
        beamline = "unknown"
        proposal = "unknown"

        # If we have a data collection id, use it
        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            identifier = str(self.dataInput.dataCollectionId.value)
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            self.edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
            directory = ispybDataCollection.imageDirectory
            template = ispybDataCollection.fileTemplate.replace("%04d", "####")
            imageNoStart = ispybDataCollection.startImageNumber
            imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1

#            # DEBUG we set the end image to 20 in order to speed up things
#            self.warning("End image set to 20 (was {0})".format(imageNoEnd))
#            imageNoEnd = 20
            pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
        else:
            identifier = str(int(time.time()))
            directory = self.dataInput.dirN.value
            template = self.dataInput.templateN.value
            imageNoStart = self.dataInput.fromN.value
            imageNoEnd = self.dataInput.toN.value
            fileTemplate = template.replace("####", "%04d")
            pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)

        # Try to get proposal from path
        if EDUtilsPath.isESRF():
            listDirectory = directory.split(os.sep)
            try:
                if listDirectory[1] == "data":
                    if listDirectory[2] == "visitor":
                        beamline = listDirectory[4]
                        proposal = listDirectory[3]
                    else:
                        beamline = listDirectory[2]
                        proposal = listDirectory[4]
            except:
                beamline = "unknown"
                proposal = userName


        if imageNoEnd - imageNoStart < 8:
            error_message = "There are fewer than 8 images, aborting"
            self.addErrorMessage(error_message)
            self.ERROR(error_message)
            self.setFailure()
            return

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        # Make results directory
        self.resultsDirectory = os.path.join(processDirectory, "results")
        if not os.path.exists(self.resultsDirectory):
            os.makedirs(self.resultsDirectory, 0755)

        # Create path to pyarch
        self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
        self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
        if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
            os.makedirs(self.pyarchDirectory, 0755)

        # Determine pyarch prefix
        listPrefix = template.split("_")
        self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])

        isH5 = False
        if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]):
            minSizeFirst = 6000000
            minSizeLast = 6000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
            minSizeFirst = 2000000
            minSizeLast = 2000000
        elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(directory,
                                            self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(directory,
                                          self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True
        else:
            minSizeFirst = 1000000
            minSizeLast = 1000000

        fWaitFileTimeout = 3600  # s

        xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
        xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
        self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
        self.edPluginWaitFileFirst.executeSynchronous()
        if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
            strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
            self.addWarningMessage(strWarningMessage)
            self.WARNING(strWarningMessage)

        xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
        xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
        self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
        self.edPluginWaitFileLast.executeSynchronous()
        if self.edPluginWaitFileLast.dataOutput.timedOut.value:
            strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
            self.addErrorMessage(strErrorMessage)
            self.ERROR(strErrorMessage)
            self.setFailure()



        # Prepare input to execution plugin
        xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
        xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
        if self.doAnomAndNonanom:
            xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
            xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
        if isH5:
            masterFilePath = os.path.join(directory,
                                          self.eiger_template_to_master(template))
            xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(masterFilePath)))
            if self.doAnomAndNonanom:
                xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(masterFilePath)))
        else:
            xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(pathToStartImage)))
            if self.doAnomAndNonanom:
                xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(pathToStartImage)))
        timeStart = time.localtime()
        self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
        self.edPluginExecXia2DIALSAnom.execute()
        if self.doAnomAndNonanom:
            self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
            self.edPluginExecXia2DIALSNoanom.execute()
        self.edPluginExecXia2DIALSAnom.synchronize()
        if self.doAnomAndNonanom:
            self.edPluginExecXia2DIALSNoanom.synchronize()
        timeEnd = time.localtime()

        # Upload to ISPyB
        self.uploadToISPyB(self.edPluginExecXia2DIALSAnom, True, proposal, timeStart, timeEnd)
        if self.doAnomAndNonanom:
            self.uploadToISPyB(self.edPluginExecXia2DIALSNoanom, False, proposal, timeStart, timeEnd)
Пример #4
0
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlCrystFELv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        userName = os.environ["USER"]

        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            #self.messenger.sendProcessingStatus(self.dataInput.dataCollectionId.value,
            #                                    "CrystFEL",
            #                                    "started")
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
            )
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.ispyb_retrieve_dc_plugin.dataInput = xsDataInputRetrieveDataCollection
            self.ispyb_retrieve_dc_plugin.executeSynchronous()
            ispybDataCollection = self.ispyb_retrieve_dc_plugin.dataOutput.dataCollection

            directory = ispybDataCollection.imageDirectory
            if EDUtilsPath.isEMBL():
                #TODO PE2 has 7 digits others 5
                template = ispybDataCollection.fileTemplate.replace(
                    "%07d", "####")
            else:
                template = ispybDataCollection.fileTemplate.replace(
                    "%04d", "####")
            imageNoStart = ispybDataCollection.startImageNumber
            imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        dir_parts = processDirectory.split("/")
        processDirectory = "/data/users/%s" % os.path.join(*dir_parts[5:])

        # Make results directory
        self.resultsDirectory = os.path.join(processDirectory,
                                             "crystfel_results")
        try:
            if not os.path.exists(self.resultsDirectory):
                os.makedirs(self.resultsDirectory)
        except Exception as ex:
            print ex

        self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
            directory)

        #TODO: enable when ready
        """
        if self.pyarchDirectory is not None:
            self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
            if not os.path.exists(self.pyarchDirectory):
                os.makedirs(self.pyarchDirectory)
        """

        self.baseFileName = "%s/crystfel_xgandalf_%d" % (
            self.resultsDirectory, self.dataInput.dataCollectionId.value)

        xsDataInputCrystFEL = XSDataInputCrystFEL()

        # Check if imagesFullPath exists. If not then generate one
        if self.dataInput.imagesFullPath is None:
            imagesFullPath = "%s_images_fullpath.lst" % self.baseName
            imagesFullFile = open(imagesFullPath, "w")
            for index in range(imageNoEnd - imageNoStart + 1):
                imagePath = os.path.join(
                    directory, ispybDataCollection.fileTemplate % (index + 1))
                imagesFullFile.write("%s\n" % imagePath)
            imagesFullFile.close()
            xsDataInputCrystFEL.imagesFullPath = XSDataString(imagesFullPath)
        else:
            xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath

        xsDataInputCrystFEL.geomFile = self.dataInput.geomFile
        xsDataInputCrystFEL.cellFile = self.dataInput.cellFile
        xsDataInputCrystFEL.pointGroup = self.dataInput.pointGroup
        xsDataInputCrystFEL.spaceGroup = self.dataInput.spaceGroup
        xsDataInputCrystFEL.resCutOff = self.dataInput.resCutOff
        xsDataInputCrystFEL.baseFileName = XSDataString(self.baseFileName)

        self.timeStart = time.localtime()

        self.integration_id, self.program_id = self.create_integration_id(
            self.dataInput.dataCollectionId.value, "Creating integration ID")

        self.index_plugin.dataInput = xsDataInputCrystFEL
        self.process_hkl_plugin.dataInput = xsDataInputCrystFEL
        self.process_hkl_plugin_odd.dataInput = xsDataInputCrystFEL
        self.process_hkl_plugin_even.dataInput = xsDataInputCrystFEL
        self.partialator_plugin.dataInput = xsDataInputCrystFEL
        self.post_process_plugin.dataInput = xsDataInputCrystFEL
        self.post_process_plugin.dataOutput = XSDataResultCrystFEL

        self.process_hkl_plugin_odd.process_hkl_options += " --odd-only"
        self.process_hkl_plugin_odd.process_hkl_type = "o"
        self.process_hkl_plugin_even.process_hkl_options += " --even-only"
        self.process_hkl_plugin_even.process_hkl_type = "e"

        self.index_plugin.executeSynchronous()
        if self.index_plugin.isFailure():
            self.ERROR('indexamajig: Failed')
            self.setFailure()
            return
        else:
            self.screen('indexamajig: Finished')

        self.process_hkl_plugin.executeSynchronous()
        if self.process_hkl_plugin.isFailure():
            self.ERROR('process_hkl: Failed')
            self.setFailure()
            return
        else:
            self.screen('process_hkl: Finished')

        self.process_hkl_plugin_odd.executeSynchronous()
        if self.process_hkl_plugin_odd.isFailure():
            self.ERROR('process_hkl_odd: Failed')
            self.setFailure()
            return
        else:
            self.screen('process_hkl_odd: Finished')

        self.process_hkl_plugin_even.executeSynchronous()
        if self.process_hkl_plugin_even.isFailure():
            self.ERROR('process_hkl_even: Failed')
            self.setFailure()
            return
        else:
            self.screen('process_hkl_even: Finished')

        self.partialator_plugin.executeSynchronous()
        if self.partialator_plugin.isFailure():
            self.ERROR('partialator: Failed')
            self.setFailure()
            return
        else:
            self.screen('partialator: Finished')

        self.post_process_plugin.executeSynchronous()
        if self.post_process_plugin.isFailure():
            self.ERROR('post_process: Failed')
            self.setFailure()
            return
        else:
            self.screen('post_process: Finished')

        self.timeEnd = time.localtime()
Пример #5
0
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlXDSAPPv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        pathToStartImage = None
        pathToEndImage = None
        userName = os.environ["USER"]
        beamline = "unknown"
        proposal = "unknown"

        if self.dataInput.startImageNumber is not None:
            imageNoStart = self.dataInput.startImageNumber.value
        if self.dataInput.endImageNumber is not None:
            imageNoEnd = self.dataInput.endImageNumber.value
        # If we have a data collection id, use it
        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            identifier = str(self.dataInput.dataCollectionId.value)
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            self.edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
            directory = ispybDataCollection.imageDirectory
            template = ispybDataCollection.fileTemplate.replace("%04d", "####")
            if imageNoStart is None:
                imageNoStart = ispybDataCollection.startImageNumber
            if imageNoEnd is None:
                imageNoEnd = ispybDataCollection.startImageNumber + \
                             ispybDataCollection.numberOfImages - 1

#            # DEBUG we set the end image to 20 in order to speed up things
#            self.warning("End image set to 20 (was {0})".format(imageNoEnd))
#            imageNoEnd = 20
            pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
        else:
            identifier = str(int(time.time()))
            directory = self.dataInput.dirN.value
            template = self.dataInput.templateN.value
            fileTemplate = template.replace("####", "%04d")
            pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)

        # Try to get proposal from path
        if EDUtilsPath.isESRF():
            listDirectory = directory.split(os.sep)
            try:
                if listDirectory[1] == "data":
                    if listDirectory[2] == "visitor":
                        beamline = listDirectory[4]
                        proposal = listDirectory[3]
                    else:
                        beamline = listDirectory[2]
                        proposal = listDirectory[4]
            except:
                beamline = "unknown"
                proposal = userName


        if imageNoEnd - imageNoStart < 8:
            error_message = "There are fewer than 8 images, aborting"
            self.addErrorMessage(error_message)
            self.ERROR(error_message)
            self.setFailure()
            return

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        # Make results directory
        self.resultsDirectory = os.path.join(processDirectory, "results")
        if not os.path.exists(self.resultsDirectory):
            os.makedirs(self.resultsDirectory, 0o755)

        # Create path to pyarch
        if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
                "XDSAPP", self.dataInput.dataCollectionId.value)
        else:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
        self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
        if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
            try:
                os.makedirs(self.pyarchDirectory, 0o755)
            except:
                self.pyarchDirectory = None

        # Determine pyarch prefix
        listPrefix = template.split("_")
        self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])

        isH5 = False
        if any(beamline in pathToStartImage for beamline in ["id30b"]):
            minSizeFirst = 6000000
            minSizeLast = 6000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
            minSizeFirst = 2000000
            minSizeLast = 2000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(directory,
                                            self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(directory,
                                          self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True
        else:
            minSizeFirst = 1000000
            minSizeLast = 1000000

        fWaitFileTimeout = 3600  # s

        xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
        xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
        self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
        self.edPluginWaitFileFirst.executeSynchronous()
        if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
            strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
            self.addWarningMessage(strWarningMessage)
            self.WARNING(strWarningMessage)

        xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
        xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
        self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
        self.edPluginWaitFileLast.executeSynchronous()
        if self.edPluginWaitFileLast.dataOutput.timedOut.value:
            strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
            self.addErrorMessage(strErrorMessage)
            self.ERROR(strErrorMessage)
            self.setFailure()


        if self.dataInput.hdf5ToCbfDirectory is not None:
            dir = self.dataInput.hdf5ToCbfDirectory.path.value
            pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0]

        self.timeStart = time.localtime()
        # Prepare input to execution plugin
        if self.doAnom:
            xsDataInputXDSAPPAnom = XSDataInputXDSAPP()
            xsDataInputXDSAPPAnom.startImageNumber = self.dataInput.startImageNumber
            xsDataInputXDSAPPAnom.endImageNumber = self.dataInput.endImageNumber
            xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True)
            xsDataInputXDSAPPAnom.image = XSDataFile(XSDataString(pathToStartImage))
            if self.xdsAppSpacegroup is not None:
                xsDataInputXDSAPPAnom.spacegroup = XSDataString(self.xdsAppSpacegroup)
            self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom
            self.edPluginExecXDSAPPAnom.execute()
            if self.dataInput.dataCollectionId is not None:
                # Set ISPyB to started
                self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingPrograms=self.processingPrograms, isAnom=True,
                                                             timeStart=self.timeStart)
        if self.doNoanom:
            xsDataInputXDSAPPNoanom = XSDataInputXDSAPP()
            xsDataInputXDSAPPNoanom.startImageNumber = self.dataInput.startImageNumber
            xsDataInputXDSAPPNoanom.endImageNumber = self.dataInput.endImageNumber
            xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False)
            xsDataInputXDSAPPNoanom.image = XSDataFile(XSDataString(pathToStartImage))
            if self.xdsAppSpacegroup is not None:
                xsDataInputXDSAPPNoanom.spacegroup = XSDataString(self.xdsAppSpacegroup)
            self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom
            self.edPluginExecXDSAPPNoanom.execute()
            if self.dataInput.dataCollectionId is not None:
                # Set ISPyB to started
                self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
                    EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                               processingPrograms=self.processingPrograms, isAnom=False,
                                                               timeStart=self.timeStart)
        if self.doAnom:
            self.edPluginExecXDSAPPAnom.synchronize()
            xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput
            # Run XSCALE even if XSCALE.LP is present
            strPathXscaleLpAnom = self.runXscale(self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True)
        if self.doNoanom:
            self.edPluginExecXDSAPPNoanom.synchronize()
            xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput
            strPathXscaleLpNoanom = self.runXscale(self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True)
        self.timeEnd = time.localtime()
        # Upload to ISPyB
        if self.dataInput.dataCollectionId is not None:
            # Check if we should use XDS_ASCII_to_XML.pl
            if self.doAnom:
                if self.useXdsAsciiToXml:
                    # Only for anom runs
                    self.runXdsAsciiToXml(xsDataResultXDSAPPAnom,
                                          self.dataInput.dataCollectionId.value,
                                          self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
                else:
                    self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPAnom, processDirectory, template,
                                       strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd,
                                       self.dataInput.dataCollectionId.value,
                                       self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
                    if self.hasUploadedAnomResultsToISPyB:
                        self.screen("Anom results uploaded to ISPyB")
                    else:
                        self.ERROR("Could not upload anom results to ISPyB!")

            if self.doNoanom:
                self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPNoanom, processDirectory, template,
                                   strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd,
                                   self.dataInput.dataCollectionId.value,
                                   self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom)
                if self.hasUploadedNoanomResultsToISPyB:
                    self.screen("Noanom results uploaded to ISPyB")
                else:
                    self.ERROR("Could not upload noanom results to ISPyB!")
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        pathToStartImage = None
        pathToEndImage = None
        userName = os.environ["USER"]
        beamline = "unknown"
        proposal = "unknown"

        # If we have a data collection id, use it
        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
            )
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            self.edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
            directory = ispybDataCollection.imageDirectory
            if EDUtilsPath.isEMBL():
                template = ispybDataCollection.fileTemplate.replace(
                    "%05d", "####")
            else:
                template = ispybDataCollection.fileTemplate.replace(
                    "%04d", "####")
            if self.dataInput.startFrame is None:
                imageNoStart = ispybDataCollection.startImageNumber
            else:
                imageNoStart = self.dataInput.startFrame.value
            if self.dataInput.endFrame is None:
                imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
            else:
                imageNoEnd = self.dataInput.endFrame.value

#            # DEBUG we set the end image to 20 in order to speed up things
#            self.warning("End image set to 20 (was {0})".format(imageNoEnd))
#            imageNoEnd = 20
            pathToStartImage = os.path.join(
                directory, ispybDataCollection.fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(
                directory, ispybDataCollection.fileTemplate % imageNoEnd)
#        else:
#            directory = self.dataInput.dirN.value
#            template = self.dataInput.templateN.value
#            imageNoStart = self.dataInput.fromN.value
#            imageNoEnd = self.dataInput.toN.value
#            fileTemplate = template.replace("####", "%04d")
#            pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
#            pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)

# Try to get proposal from path
        if EDUtilsPath.isESRF():
            listDirectory = directory.split(os.sep)
            try:
                if listDirectory[1] == "data":
                    if listDirectory[2] == "visitor":
                        beamline = listDirectory[4]
                        proposal = listDirectory[3]
                    else:
                        beamline = listDirectory[2]
                        proposal = listDirectory[4]
            except:
                beamline = "unknown"
                proposal = userName

        if imageNoEnd - imageNoStart < 8:
            error_message = "There are fewer than 8 images, aborting"
            self.addErrorMessage(error_message)
            self.ERROR(error_message)
            self.setFailure()
            return

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        # Make results directory
        self.resultsDirectory = os.path.join(processDirectory, "results")
        if not os.path.exists(self.resultsDirectory):
            os.makedirs(self.resultsDirectory, 0o755)

        # Create path to pyarch
        if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
                beamline, "XIA2_DIALS", self.dataInput.dataCollectionId.value)
        else:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
                self.resultsDirectory)
        if self.pyarchDirectory is not None:
            self.pyarchDirectory = self.pyarchDirectory.replace(
                'PROCESSED_DATA', 'RAW_DATA')
            if not os.path.exists(self.pyarchDirectory):
                try:
                    os.makedirs(self.pyarchDirectory, 0o755)
                except:
                    self.pyarchDirectory = None

        # Determine pyarch prefix
        listPrefix = template.split("_")
        self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3],
                                                   listPrefix[-2])

        isH5 = False
        if any(beamline in pathToStartImage
               for beamline in ["id23eh1", "id29"]):
            minSizeFirst = 6000000
            minSizeLast = 6000000
        elif any(beamline in pathToStartImage
                 for beamline in ["id23eh2", "id30a1"]):
            minSizeFirst = 2000000
            minSizeLast = 2000000
        elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(
                directory,
                self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(
                directory, self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True
        else:
            minSizeFirst = 1000000
            minSizeLast = 1000000

        if EDUtilsPath.isEMBL():
            fWaitFileTimeout = 60
        else:
            fWaitFileTimeout = 3600  # s

        xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileFirst.file = XSDataFile(
            XSDataString(pathToStartImage))
        xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
        self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
        self.edPluginWaitFileFirst.executeSynchronous()
        if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
            strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
                fWaitFileTimeout, pathToStartImage)
            self.addWarningMessage(strWarningMessage)
            self.WARNING(strWarningMessage)

        xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileLast.file = XSDataFile(
            XSDataString(pathToEndImage))
        xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
        self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
        self.edPluginWaitFileLast.executeSynchronous()
        if self.edPluginWaitFileLast.dataOutput.timedOut.value:
            strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (
                fWaitFileTimeout, pathToEndImage)
            self.addErrorMessage(strErrorMessage)
            self.ERROR(strErrorMessage)
            self.setFailure()

        # Prepare input to execution plugin
        if self.doAnom:
            xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
            xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
            xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup
            xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell
            if imageNoStart is not None:
                xsDataInputXia2DIALSAnom.startFrame = XSDataInteger(
                    imageNoStart)
            if imageNoEnd is not None:
                xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd)
        if self.doNoanom:
            xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
            xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
            xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup
            xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell
            if imageNoStart is not None:
                xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger(
                    imageNoStart)
            if imageNoEnd is not None:
                xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd)
        if isH5:
            masterFilePath = os.path.join(
                directory, self.eiger_template_to_master(template))
            if self.doAnom:
                xsDataInputXia2DIALSAnom.addImage(
                    XSDataFile(XSDataString(masterFilePath)))
            if self.doNoanom:
                xsDataInputXia2DIALSNoanom.addImage(
                    XSDataFile(XSDataString(masterFilePath)))
        else:
            if self.doAnom:
                xsDataInputXia2DIALSAnom.addImage(
                    XSDataFile(XSDataString(pathToStartImage)))
            if self.doNoanom:
                xsDataInputXia2DIALSNoanom.addImage(
                    XSDataFile(XSDataString(pathToStartImage)))
        self.timeStart = time.localtime()

        if self.dataInput.dataCollectionId is not None:
            # Set ISPyB to running
            if self.doAnom:
                self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingPrograms,
                                                             isAnom=True,
                                                             timeStart=self.timeStart)
            if self.doNoanom:
                self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingPrograms,
                                                             isAnom=False,
                                                             timeStart=self.timeStart)

        if self.doAnom:
            self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
            self.edPluginExecXia2DIALSAnom.execute()
        if self.doNoanom:
            self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
            self.edPluginExecXia2DIALSNoanom.execute()
        if self.doAnom:
            self.edPluginExecXia2DIALSAnom.synchronize()
        if self.doNoanom:
            self.edPluginExecXia2DIALSNoanom.synchronize()
        self.timeEnd = time.localtime()

        # Upload to ISPyB
        if self.doAnom:
            self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(
                self.edPluginExecXia2DIALSAnom, True, proposal,
                self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom)
            if self.hasUploadedAnomResultsToISPyB:
                self.screen("Anom results uploaded to ISPyB")
            else:
                self.ERROR("Could not upload anom results to ISPyB!")
        if self.doNoanom:
            self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(
                self.edPluginExecXia2DIALSNoanom, False, proposal,
                self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom)
            if self.hasUploadedNoanomResultsToISPyB:
                self.screen("Noanom results uploaded to ISPyB")
            else:
                self.ERROR("Could not upload noanom results to ISPyB!")
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlXDSAPPv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        pathToStartImage = None
        pathToEndImage = None
        userName = os.environ["USER"]
        beamline = "unknown"
        proposal = "unknown"

        # If we have a data collection id, use it
        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            identifier = str(self.dataInput.dataCollectionId.value)
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            self.edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
            directory = ispybDataCollection.imageDirectory
            template = ispybDataCollection.fileTemplate.replace("%04d", "####")
            imageNoStart = ispybDataCollection.startImageNumber
            imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1

#            # DEBUG we set the end image to 20 in order to speed up things
#            self.warning("End image set to 20 (was {0})".format(imageNoEnd))
#            imageNoEnd = 20
            pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
        else:
            identifier = str(int(time.time()))
            directory = self.dataInput.dirN.value
            template = self.dataInput.templateN.value
            imageNoStart = self.dataInput.fromN.value
            imageNoEnd = self.dataInput.toN.value
            fileTemplate = template.replace("####", "%04d")
            pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)

        # Try to get proposal from path
        if EDUtilsPath.isESRF():
            listDirectory = directory.split(os.sep)
            try:
                if listDirectory[1] == "data":
                    if listDirectory[2] == "visitor":
                        beamline = listDirectory[4]
                        proposal = listDirectory[3]
                    else:
                        beamline = listDirectory[2]
                        proposal = listDirectory[4]
            except:
                beamline = "unknown"
                proposal = userName


        if imageNoEnd - imageNoStart < 8:
            error_message = "There are fewer than 8 images, aborting"
            self.addErrorMessage(error_message)
            self.ERROR(error_message)
            self.setFailure()
            return

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        # Make results directory
        self.resultsDirectory = os.path.join(processDirectory, "results")
        if not os.path.exists(self.resultsDirectory):
            os.makedirs(self.resultsDirectory, 0o755)

        isH5 = False
        if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]):
            minSizeFirst = 6000000
            minSizeLast = 6000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
            minSizeFirst = 2000000
            minSizeLast = 2000000
        elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(directory,
                                            self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(directory,
                                          self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True
        else:
            minSizeFirst = 1000000
            minSizeLast = 1000000

        fWaitFileTimeout = 3600  # s

        xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
        xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
        self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
        self.edPluginWaitFileFirst.executeSynchronous()
        if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
            strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
            self.addWarningMessage(strWarningMessage)
            self.WARNING(strWarningMessage)

        xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
        xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
        self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
        self.edPluginWaitFileLast.executeSynchronous()
        if self.edPluginWaitFileLast.dataOutput.timedOut.value:
            strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
            self.addErrorMessage(strErrorMessage)
            self.ERROR(strErrorMessage)
            self.setFailure()


        if self.dataInput.hdf5ToCbfDirectory is not None:
            dir = self.dataInput.hdf5ToCbfDirectory.path.value
            pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0]

        # Prepare input to execution plugin
        xsDataInputXDSAPPAnom = XSDataInputXDSAPP()
        xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True)
        xsDataInputXDSAPPAnom.image = XSDataFile(XSDataString(pathToStartImage))
        if self.xdsAppSpacegroup is not None:
            xsDataInputXDSAPPAnom.spacegroup = XSDataString(self.xdsAppSpacegroup)
        if self.doAnomAndNonanom:
            xsDataInputXDSAPPNoanom = XSDataInputXDSAPP()
            xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False)
            xsDataInputXDSAPPNoanom.image = XSDataFile(XSDataString(pathToStartImage))
            if self.xdsAppSpacegroup is not None:
                xsDataInputXDSAPPNoanom.spacegroup = XSDataString(self.xdsAppSpacegroup)
#        if isH5:
#            masterFilePath = os.path.join(directory,
#                                          self.eiger_template_to_master(template))
#            xsDataInputXDSAPPAnom.addImage(XSDataFile(XSDataString(masterFilePath)))
#            if self.doAnomAndNonanom:
#                xsDataInputXDSAPPNoanom.addImage(XSDataFile(XSDataString(masterFilePath)))
#        else:
#            xsDataInputXDSAPPAnom.addImage(XSDataFile(XSDataString(pathToStartImage)))
#            if self.doAnomAndNonanom:
#                xsDataInputXDSAPPNoanom.addImage(XSDataFile(XSDataString(pathToStartImage)))
        self.timeStart = time.localtime()
        self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom
        self.edPluginExecXDSAPPAnom.execute()
        if self.dataInput.dataCollectionId is not None:
            # Set ISPyB to started
            self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
              EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                         processingPrograms=self.processingPrograms, isAnom=True,
                                                         timeStart=self.timeStart)

        if self.doAnomAndNonanom:
            self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom
            self.edPluginExecXDSAPPNoanom.execute()
            if self.dataInput.dataCollectionId is not None:
                # Set ISPyB to started
                self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
                    EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                               processingPrograms=self.processingPrograms, isAnom=False,
                                                               timeStart=self.timeStart)
        self.edPluginExecXDSAPPAnom.synchronize()
        xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput
        # Run XSCALE even if XSCALE.LP is present
        strPathXscaleLpAnom = self.runXscale(self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True)
        if self.doAnomAndNonanom:
            self.edPluginExecXDSAPPNoanom.synchronize()
            xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput
            strPathXscaleLpNoanom = self.runXscale(self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True)
        self.timeEnd = time.localtime()
        # Upload to ISPyB
        if self.dataInput.dataCollectionId is not None:
            # Check if we should use XDS_ASCII_to_XML.pl
            if self.useXdsAsciiToXml:
                # Only for anom runs
                self.runXdsAsciiToXml(xsDataResultXDSAPPAnom,
                                      self.dataInput.dataCollectionId.value,
                                      self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
            else:
                self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPAnom, processDirectory, template,
                                   strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd,
                                   self.dataInput.dataCollectionId.value,
                                   self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
                if self.hasUploadedAnomResultsToISPyB:
                    self.screen("Anom results uploaded to ISPyB")
                else:
                    self.ERROR("Could not upload anom results to ISPyB!")



                if self.doAnomAndNonanom:
                    self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPNoanom, processDirectory, template,
                                       strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd,
                                       self.dataInput.dataCollectionId.value,
                                       self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom)
                    if self.hasUploadedNoanomResultsToISPyB:
                        self.screen("Noanom results uploaded to ISPyB")
                    else:
                        self.ERROR("Could not upload noanom results to ISPyB!")
Пример #8
0
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlDozorv1_0.process")
        self.sendMessageToMXCuBE("Processing started...", "info")
        EDUtilsParallel.initializeNbThread()
        xsDataResultControlDozor = XSDataResultControlDozor()
        # Check if connection to ISPyB needed
        if self.dataInput.dataCollectionId is not None:
            edPluginRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = edPluginRetrieveDataCollection.dataOutput.dataCollection
            if self.batchSize is None:
                batchSize = ispybDataCollection.numberOfImages
            else:
                batchSize = self.batchSize
            if batchSize > self.maxBatchSize:
                batchSize = self.maxBatchSize
            if abs(ispybDataCollection.overlap) > 1:
                self.hasOverlap = True
                self.overlap = ispybDataCollection.overlap
            dictImage = self.createImageDictFromISPyB(ispybDataCollection)
        else:
            # No connection to ISPyB, take parameters from input
            if self.dataInput.batchSize is None:
                batchSize = self.maxBatchSize
            else:
                batchSize = self.dataInput.batchSize.value
            dictImage = self.createImageDict(self.dataInput)
        self.screen("Dozor batch size: {0}".format(batchSize))
        if self.dataInput.hdf5BatchSize is not None:
            self.hdf5BatchSize = self.dataInput.hdf5BatchSize.value
        listAllBatches = self.createListOfBatches(dictImage.keys(), batchSize)
        if dictImage[listAllBatches[0][0]].path.value.endswith("h5"):
            # Convert HDF5 images to CBF
            self.screen("HDF5 converter batch size: {0}".format(self.batchSize))
            if self.doRadiationDamage:
                self.cbfTempDir = None
            else:
                self.cbfTempDir = tempfile.mkdtemp(prefix="CbfTemp_")
            listHdf5Batches = self.createListOfBatches(dictImage.keys(), self.batchSize)
            dictImage, self.hasHdf5Prefix = self.convertToCBF(dictImage, listHdf5Batches, self.doRadiationDamage)
        for listBatch in listAllBatches:
            # Read the header from the first image in the batch
            xsDataFile = dictImage[listBatch[0]]
            edPluginControlReadImageHeader = self.loadPlugin(self.strEDPluginControlReadImageHeaderName)
            xsDataInputReadImageHeader = XSDataInputReadImageHeader()
            xsDataInputReadImageHeader.image = xsDataFile
            edPluginControlReadImageHeader.dataInput = xsDataInputReadImageHeader
            edPluginControlReadImageHeader.executeSynchronous()
            subWedge = edPluginControlReadImageHeader.dataOutput.subWedge
            xsDataInputDozor = XSDataInputDozor()
            beam = subWedge.experimentalCondition.beam
            detector = subWedge.experimentalCondition.detector
            goniostat = subWedge.experimentalCondition.goniostat
            xsDataInputDozor.detectorType = detector.type
            xsDataInputDozor.exposureTime = XSDataDouble(beam.exposureTime.value)
            xsDataInputDozor.spotSize = XSDataInteger(3)
            xsDataInputDozor.detectorDistance = XSDataDouble(detector.distance.value)
            xsDataInputDozor.wavelength = XSDataDouble(beam.wavelength.value)
#            xsDataInputDozor.fractionPolatization : XSDataDouble optional
            orgx = detector.beamPositionY.value / detector.pixelSizeY.value
            orgy = detector.beamPositionX.value / detector.pixelSizeX.value
            xsDataInputDozor.orgx = XSDataDouble(orgx)
            xsDataInputDozor.orgy = XSDataDouble(orgy)
            xsDataInputDozor.oscillationRange = XSDataDouble(goniostat.oscillationWidth.value)
#            xsDataInputDozor.imageStep : XSDataDouble optional
            xsDataInputDozor.startingAngle = XSDataDouble(goniostat.rotationAxisStart.value)
            xsDataInputDozor.firstImageNumber = subWedge.image[0].number
            xsDataInputDozor.numberImages = XSDataInteger(len(listBatch))
            if self.hasOverlap:
                xsDataInputDozor.overlap = XSDataAngle(self.overlap)
            strFileName = subWedge.image[0].path.value
            strPrefix = EDUtilsImage.getPrefix(strFileName)
            strSuffix = EDUtilsImage.getSuffix(strFileName)
            if EDUtilsPath.isEMBL():
                strXDSTemplate = "%s_?????.%s" % (strPrefix, strSuffix)
            elif self.hasHdf5Prefix and not self.hasOverlap:
                strXDSTemplate = "%s_??????.%s" % (strPrefix, strSuffix)
            else:
                strXDSTemplate = "%s_????.%s" % (strPrefix, strSuffix)
            xsDataInputDozor.nameTemplateImage = XSDataString(os.path.join(os.path.dirname(strFileName), strXDSTemplate))
            xsDataInputDozor.wedgeNumber = self.dataInput.wedgeNumber
            xsDataInputDozor.radiationDamage = self.dataInput.radiationDamage
            edPluginDozor = self.loadPlugin(self.strEDPluginDozorName, "Dozor_%05d" % subWedge.image[0].number.value)
            edPluginDozor.dataInput = xsDataInputDozor
            edPluginDozor.execute()
            edPluginDozor.synchronize()
            indexImage = 0
            imageDozorBatchList = []

            for xsDataResultDozor in edPluginDozor.dataOutput.imageDozor:
                xsDataControlImageDozor = XSDataControlImageDozor()
                xsDataControlImageDozor.number = xsDataResultDozor.number
                xsDataControlImageDozor.image = dictImage[listBatch[indexImage]]
                xsDataControlImageDozor.spotsNumOf = xsDataResultDozor.spotsNumOf
                xsDataControlImageDozor.spotsIntAver = xsDataResultDozor.spotsIntAver
                xsDataControlImageDozor.spotsResolution = xsDataResultDozor.spotsResolution
                xsDataControlImageDozor.powderWilsonScale = xsDataResultDozor.powderWilsonScale
                xsDataControlImageDozor.powderWilsonBfactor = xsDataResultDozor.powderWilsonBfactor
                xsDataControlImageDozor.powderWilsonResolution = xsDataResultDozor.powderWilsonResolution
                xsDataControlImageDozor.powderWilsonCorrelation = xsDataResultDozor.powderWilsonCorrelation
                xsDataControlImageDozor.powderWilsonRfactor = xsDataResultDozor.powderWilsonRfactor
                xsDataControlImageDozor.mainScore = xsDataResultDozor.mainScore
                xsDataControlImageDozor.spotScore = xsDataResultDozor.spotScore
                xsDataControlImageDozor.visibleResolution = xsDataResultDozor.visibleResolution
                xsDataControlImageDozor.spotFile = xsDataResultDozor.spotFile
                xsDataControlImageDozor.angle = xsDataResultDozor.angle
                xsDataResultControlDozor.addImageDozor(xsDataControlImageDozor)
                if xsDataResultControlDozor.inputDozor is None:
                    xsDataResultControlDozor.inputDozor = XSDataDozorInput().parseString(xsDataInputDozor.marshal())
                indexImage += 1

                dozorSpotListShape = []
                dozorSpotList = []
                spotFile = None
                if xsDataControlImageDozor.spotFile is not None:
                    spotFile = xsDataControlImageDozor.spotFile.path.value
                    if os.path.exists(spotFile):
                        numpyArray = numpy.loadtxt(spotFile, skiprows=3)
                        dozorSpotList = base64.b64encode(numpyArray.tostring())
                        dozorSpotListShape.append(numpyArray.shape[0])
                        if len(numpyArray.shape) > 1:
                            dozorSpotListShape.append(numpyArray.shape[1])

                imageDozorDict = {"index": xsDataControlImageDozor.number.value,
                                  "imageName": xsDataControlImageDozor.image.path.value,
                                  "dozor_score": xsDataControlImageDozor.mainScore.value,
                                  "dozorSpotsNumOf" : xsDataControlImageDozor.spotsNumOf.value,
                                  "dozorSpotFile": spotFile,
                                  "dozorSpotList" : dozorSpotList,
                                  "dozorSpotListShape": dozorSpotListShape,
                                  "dozorSpotsIntAver": xsDataControlImageDozor.spotsIntAver.value,
                                  "dozorSpotsResolution": xsDataControlImageDozor.spotsResolution.value
                                  }
                imageDozorBatchList.append(imageDozorDict)

            xsDataResultControlDozor.halfDoseTime = edPluginDozor.dataOutput.halfDoseTime
            xsDataResultControlDozor.pngPlots = edPluginDozor.dataOutput.pngPlots

            self.sendResultToMXCuBE(imageDozorBatchList)
            self.sendMessageToMXCuBE("Batch processed")
        self.dataOutput = xsDataResultControlDozor
        if self.cbfTempDir is not None:
            if self.dataInput.keepCbfTmpDirectory is not None and self.dataInput.keepCbfTmpDirectory.value:
                self.dataOutput.pathToCbfDirectory = XSDataFile(XSDataString(self.cbfTempDir))
            else:
                shutil.rmtree(self.cbfTempDir)
 def process(self, _edObject=None):
     EDPluginControl.process(self)
     self.DEBUG("EDPluginControlDozorv1_0.process")
     xsDataResultControlDozor = XSDataResultControlDozor()
     # Check if connection to ISPyB needed
     if self.dataInput.dataCollectionId is not None:
         edPluginRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
         xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
         xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
         edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
         edPluginRetrieveDataCollection.executeSynchronous()
         ispybDataCollection = edPluginRetrieveDataCollection.dataOutput.dataCollection
         batchSize = ispybDataCollection.numberOfImages
         if batchSize > self.maxBatchSize:
             batchSize = self.maxBatchSize
         if abs(ispybDataCollection.overlap) > 1:
             self.hasOverlap = True
             self.overlap = ispybDataCollection.overlap
         dictImage = self.createImageDictFromISPyB(ispybDataCollection)
     else:
         # No connection to ISPyB, take parameters from input
         if self.dataInput.batchSize is None:
             batchSize = 1
         else:
             batchSize = self.dataInput.batchSize.value
         dictImage = self.createImageDict(self.dataInput)
     listAllBatches = self.createListOfBatches(dictImage.keys(), batchSize)
     if dictImage[listAllBatches[0][0]].path.value.endswith("h5"):
         # Convert HDF5 images to CBF
         self.cbfTempDir = tempfile.mkdtemp(prefix="CbfTemp_")
         dictImage, self.hasHdf5Prefix = self.convertToCBF(dictImage)
     for listBatch in listAllBatches:
         # Read the header from the first image in the batch
         xsDataFile = dictImage[listBatch[0]]
         edPluginControlReadImageHeader = self.loadPlugin(self.strEDPluginControlReadImageHeaderName)
         xsDataInputReadImageHeader = XSDataInputReadImageHeader()
         xsDataInputReadImageHeader.image = xsDataFile
         edPluginControlReadImageHeader.dataInput = xsDataInputReadImageHeader
         edPluginControlReadImageHeader.executeSynchronous()
         subWedge = edPluginControlReadImageHeader.dataOutput.subWedge
         xsDataInputDozor = XSDataInputDozor()
         beam = subWedge.experimentalCondition.beam
         detector = subWedge.experimentalCondition.detector
         goniostat = subWedge.experimentalCondition.goniostat
         xsDataInputDozor.detectorType = detector.type
         xsDataInputDozor.exposureTime = XSDataDouble(beam.exposureTime.value)
         xsDataInputDozor.spotSize = XSDataInteger(3)
         xsDataInputDozor.detectorDistance = XSDataDouble(detector.distance.value)
         xsDataInputDozor.wavelength = XSDataDouble(beam.wavelength.value)
         #            xsDataInputDozor.fractionPolatization : XSDataDouble optional
         orgx = detector.beamPositionY.value / detector.pixelSizeY.value
         orgy = detector.beamPositionX.value / detector.pixelSizeX.value
         xsDataInputDozor.orgx = XSDataDouble(orgx)
         xsDataInputDozor.orgy = XSDataDouble(orgy)
         xsDataInputDozor.oscillationRange = XSDataDouble(goniostat.oscillationWidth.value)
         #            xsDataInputDozor.imageStep : XSDataDouble optional
         xsDataInputDozor.startingAngle = XSDataDouble(goniostat.rotationAxisStart.value)
         xsDataInputDozor.firstImageNumber = subWedge.image[0].number
         xsDataInputDozor.numberImages = XSDataInteger(len(listBatch))
         if self.hasOverlap:
             xsDataInputDozor.overlap = XSDataAngle(self.overlap)
         strFileName = subWedge.image[0].path.value
         strPrefix = EDUtilsImage.getPrefix(strFileName)
         strSuffix = EDUtilsImage.getSuffix(strFileName)
         if EDUtilsPath.isEMBL():
             strXDSTemplate = "%s_?????.%s" % (strPrefix, strSuffix)
         elif self.hasHdf5Prefix and not self.hasOverlap:
             strXDSTemplate = "%s_??????.%s" % (strPrefix, strSuffix)
         else:
             strXDSTemplate = "%s_????.%s" % (strPrefix, strSuffix)
         xsDataInputDozor.nameTemplateImage = XSDataString(
             os.path.join(os.path.dirname(strFileName), strXDSTemplate)
         )
         xsDataInputDozor.wedgeNumber = self.dataInput.wedgeNumber
         xsDataInputDozor.radiationDamage = self.dataInput.radiationDamage
         edPluginDozor = self.loadPlugin(self.strEDPluginDozorName, "Dozor_%05d" % subWedge.image[0].number.value)
         edPluginDozor.dataInput = xsDataInputDozor
         edPluginDozor.executeSynchronous()
         indexImage = 0
         for xsDataResultDozor in edPluginDozor.dataOutput.imageDozor:
             xsDataControlImageDozor = XSDataControlImageDozor()
             xsDataControlImageDozor.number = xsDataResultDozor.number
             xsDataControlImageDozor.image = dictImage[listBatch[indexImage]]
             xsDataControlImageDozor.spotsNumOf = xsDataResultDozor.spotsNumOf
             xsDataControlImageDozor.spotsIntAver = xsDataResultDozor.spotsIntAver
             xsDataControlImageDozor.spotsResolution = xsDataResultDozor.spotsResolution
             xsDataControlImageDozor.powderWilsonScale = xsDataResultDozor.powderWilsonScale
             xsDataControlImageDozor.powderWilsonBfactor = xsDataResultDozor.powderWilsonBfactor
             xsDataControlImageDozor.powderWilsonResolution = xsDataResultDozor.powderWilsonResolution
             xsDataControlImageDozor.powderWilsonCorrelation = xsDataResultDozor.powderWilsonCorrelation
             xsDataControlImageDozor.powderWilsonRfactor = xsDataResultDozor.powderWilsonRfactor
             xsDataControlImageDozor.mainScore = xsDataResultDozor.mainScore
             xsDataControlImageDozor.spotScore = xsDataResultDozor.spotScore
             xsDataControlImageDozor.visibleResolution = xsDataResultDozor.visibleResolution
             xsDataControlImageDozor.spotFile = xsDataResultDozor.spotFile
             xsDataControlImageDozor.angle = xsDataResultDozor.angle
             xsDataResultControlDozor.addImageDozor(xsDataControlImageDozor)
             if xsDataResultControlDozor.inputDozor is None:
                 xsDataResultControlDozor.inputDozor = XSDataDozorInput().parseString(xsDataInputDozor.marshal())
             indexImage += 1
         xsDataResultControlDozor.halfDoseTime = edPluginDozor.dataOutput.halfDoseTime
     self.dataOutput = xsDataResultControlDozor
     if self.cbfTempDir is not None:
         shutil.rmtree(self.cbfTempDir)
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlAutoPROCv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        pathToStartImage = None
        pathToEndImage = None
        userName = os.environ["USER"]
        beamline = "unknown"
        proposal = "unknown"

        # If we have a data collection id, use it
        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            identifier = str(self.dataInput.dataCollectionId.value)
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            self.edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
            directory = ispybDataCollection.imageDirectory
            if EDUtilsPath.isEMBL():
                template = ispybDataCollection.fileTemplate.replace("%05d", "#" * 5)
            elif EDUtilsPath.isMAXIV():
                template = ispybDataCollection.fileTemplate
            else:
                template = ispybDataCollection.fileTemplate.replace("%04d", "####")
            if self.dataInput.fromN is None:
                imageNoStart = ispybDataCollection.startImageNumber
            else:
                imageNoStart = self.dataInput.fromN.value
            if self.dataInput.toN is None:
                imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
            else:
                imageNoEnd = self.dataInput.toN.value
#            # DEBUG we set the end image to 20 in order to speed up things
#            self.warning("End image set to 20 (was {0})".format(imageNoEnd))
#            imageNoEnd = 20
            pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
        else:
            identifier = str(int(time.time()))
            directory = self.dataInput.dirN.path.value
            template = self.dataInput.templateN.value
            imageNoStart = self.dataInput.fromN.value
            imageNoEnd = self.dataInput.toN.value
            if EDUtilsPath.isEMBL():
                fileTemplate = template.replace("#####", "%05d")
            else:
                fileTemplate = template.replace("####", "%04d")
            pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)

        # Try to get proposal from path
        if EDUtilsPath.isESRF():
            listDirectory = directory.split(os.sep)
            try:
                if listDirectory[1] == "data":
                    if listDirectory[2] == "visitor":
                        beamline = listDirectory[4]
                        proposal = listDirectory[3]
                    else:
                        beamline = listDirectory[2]
                        proposal = listDirectory[4]
            except:
                beamline = "unknown"
                proposal = userName


        if imageNoEnd - imageNoStart < 8:
            error_message = "There are fewer than 8 images, aborting"
            self.addErrorMessage(error_message)
            self.ERROR(error_message)
            self.setFailure()
            return

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        # Make results directory
        if EDUtilsPath.isALBA():
            _processDirectory = "_".join(pathToStartImage.split('_')[:-1])
            from datetime import datetime
            _id = datetime.now().strftime('%Y%m%d_%H%M%S')
            self.resultsDirectory = os.path.join(_processDirectory, "autoPROC_%s" % _id)
        else:
            self.resultsDirectory = os.path.join(processDirectory, "results")
            if not os.path.exists(self.resultsDirectory):
                os.makedirs(self.resultsDirectory, 0o755)


        # Create path to pyarch
        if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
                "autoPROC", self.dataInput.dataCollectionId.value)
        else:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
        if self.pyarchDirectory is not None:
            self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
            if not os.path.exists(self.pyarchDirectory):
                try:
                    os.makedirs(self.pyarchDirectory, 0o755)
                except:
                    self.pyarchDirectory = None

        # The resultsDirectory is not used at ALBA (only pyarchDirectory)
        if EDUtilsPath.isALBA():
            self.resultsDirectory = None

        # Determine pyarch prefix
        if EDUtilsPath.isALBA():
            listPrefix = template.split("_")
            self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]),
                                                       listPrefix[-2])
        else:
            listPrefix = template.split("_")
            self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])

        isH5 = False
        if any(beamline in pathToStartImage for beamline in ["id30b"]):
            minSizeFirst = 6000000
            minSizeLast = 6000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
            minSizeFirst = 2000000
            minSizeLast = 2000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(directory,
                                            self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(directory,
                                          self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True
        else:
            minSizeFirst = 1000000
            minSizeLast = 1000000

        if EDUtilsPath.isMAXIV():
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(directory,
                                            self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(directory,
                                          self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True

        if EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV():
            fWaitFileTimeout = 60  # s
        else:
            fWaitFileTimeout = 3600  # s

        xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
        xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
        self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
        self.edPluginWaitFileFirst.executeSynchronous()
        if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
            strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
            self.addWarningMessage(strWarningMessage)
            self.WARNING(strWarningMessage)

        xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
        xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
        self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
        self.edPluginWaitFileLast.executeSynchronous()
        if self.edPluginWaitFileLast.dataOutput.timedOut.value:
            strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
            self.addErrorMessage(strErrorMessage)
            self.ERROR(strErrorMessage)
            self.setFailure()

        self.timeStart = time.localtime()
        if self.dataInput.dataCollectionId is not None:
            # Set ISPyB to running
            if self.doAnom:
                self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingProgram,
                                                             isAnom=True,
                                                             timeStart=self.timeStart)
                self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingProgramStaraniso,
                                                             isAnom=True,
                                                             timeStart=self.timeStart)
            if self.doNoanom:
                self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingProgram,
                                                             isAnom=False,
                                                             timeStart=self.timeStart)
                self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingProgramStaraniso,
                                                             isAnom=False,
                                                             timeStart=self.timeStart)


        # Prepare input to execution plugin
        if self.doAnom:
            xsDataInputAutoPROCAnom = XSDataInputAutoPROC()
            xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True)
            xsDataInputAutoPROCAnom.symm = self.dataInput.symm
            xsDataInputAutoPROCAnom.cell = self.dataInput.cell
            xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit
            xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit
        if self.doNoanom:
            xsDataInputAutoPROCNoanom = XSDataInputAutoPROC()
            xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(False)
            xsDataInputAutoPROCNoanom.symm = self.dataInput.symm
            xsDataInputAutoPROCNoanom.cell = self.dataInput.cell
            xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit
            xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit
        xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier()
        xsDataAutoPROCIdentifier.idN = XSDataString(identifier)
        xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory))
        xsDataAutoPROCIdentifier.templateN = XSDataString(template)
        xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart)
        xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd)
        if self.doAnom:
            xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier)
        if self.doNoanom:
            xsDataInputAutoPROCNoanom.addIdentifier(xsDataAutoPROCIdentifier.copy())
        if isH5:
            masterFilePath = os.path.join(directory,
                                          self.eiger_template_to_master(template))
            if self.doAnom:
                xsDataInputAutoPROCAnom.masterH5 = XSDataFile(XSDataString(masterFilePath))
            if self.doNoanom:
                xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(XSDataString(masterFilePath))
        timeStart = time.localtime()
        if self.doAnom:
            self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom
            self.edPluginExecAutoPROCAnom.execute()
        if self.doNoanom:
            self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom
            self.edPluginExecAutoPROCNoanom.execute()
        if self.doAnom:
            self.edPluginExecAutoPROCAnom.synchronize()
        if self.doNoanom:
            self.edPluginExecAutoPROCNoanom.synchronize()
        timeEnd = time.localtime()

        # Upload to ISPyB
        if self.doAnom:
            self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False, proposal, timeStart, timeEnd)
            self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True, proposal, timeStart, timeEnd)
        if self.doNoanom:
            self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, False, proposal, timeStart, timeEnd)
            self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, True, proposal, timeStart, timeEnd)
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlCrystFELv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        pathToStartImage = None
        pathToEndImage = None
        userName = os.environ["USER"]
        beamline = "unknown"
        proposal = "unknown"

        # If we have a data collection id, use it
        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            #self.messenger.sendProcessingStatus(self.dataInput.dataCollectionId.value,
            #                                    "CrystFEL",
            #                                    "started")
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.ispyb_retrieve_dc_plugin.dataInput = xsDataInputRetrieveDataCollection
            self.ispyb_retrieve_dc_plugin.executeSynchronous()
            ispybDataCollection = self.ispyb_retrieve_dc_plugin.dataOutput.dataCollection

            directory = ispybDataCollection.imageDirectory
            if EDUtilsPath.isEMBL():
                #TODO PE2 has 7 digits others 5
                template = ispybDataCollection.fileTemplate.replace("%07d", "####")
            else:
                template = ispybDataCollection.fileTemplate.replace("%04d", "####")
            imageNoStart = ispybDataCollection.startImageNumber
            imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1

            pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        # Make results directory
        self.resultsDirectory = os.path.join(processDirectory, "results")
        try:
            if not os.path.exists(self.resultsDirectory):
                os.makedirs(self.resultsDirectory, 755) 
        except Exception as ex:
            print ex 

        # Create path to pyarch
        self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)

        #TODO: enable when ready
        """
        if self.pyarchDirectory is not None:
            self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
            if not os.path.exists(self.pyarchDirectory):
                os.makedirs(self.pyarchDirectory, 755)
        """

        # Determine pyarch prefix
        listPrefix = template.split("_")
        self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])

        isH5 = False

        minSizeFirst = 1000000
        minSizeLast = 1000000
        fWaitFileTimeout = 60

        xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
        xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
        self.wait_first_image_plugin.size = XSDataInteger(minSizeFirst)
        self.wait_first_image_plugin.dataInput = xsDataInputMXWaitFileFirst
        self.wait_first_image_plugin.executeSynchronous()
        if self.wait_first_image_plugin.dataOutput.timedOut.value:
            strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
            self.addWarningMessage(strWarningMessage)
            self.WARNING(strWarningMessage)

        self.baseName = "%s/crystfel_xgandalf" % self.resultsDirectory

        self.streamFilename = "%s_%d.stream" % (
            self.baseName,
            self.dataInput.dataCollectionId.value
        )
        self.hklFilename = "%s_%d.hkl" % (
            self.baseName,
            self.dataInput.dataCollectionId.value
        )
        self.mtzFilename = "%s_%d.mtz" % (
            self.baseName,
            self.dataInput.dataCollectionId.value
        )

        # Prepare input to execution plugin
        xsDataInputCrystFEL = XSDataInputCrystFEL()
        # Check if imagesFullPath exists. If not then generate one
        if self.dataInput.imagesFullPath is None:
            imagesFullPath = "%s_images_fullpath.lst" % self.baseName
            imagesFullFile = open(imagesFullPath, "w")
            for index in range(imageNoEnd - imageNoStart + 1):
                imagePath = os.path.join(directory, ispybDataCollection.fileTemplate % (index + 1))
                imagesFullFile.write("%s\n" % imagePath)
            imagesFullFile.close()
            xsDataInputCrystFEL.imagesFullPath = XSDataString(imagesFullPath)
        else:
            xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath

        xsDataInputCrystFEL.geomFile = self.dataInput.geomFile
        xsDataInputCrystFEL.cellFile = self.dataInput.cellFile

        #TODO EDNA can generate the imagesFull path if id do not exist
        xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath
        xsDataInputCrystFEL.streamFile = XSDataString(self.streamFilename)
        xsDataInputCrystFEL.hklFile = XSDataString(self.hklFilename)
        xsDataInputCrystFEL.mtzFile = XSDataString(self.mtzFilename)

        self.timeStart = time.localtime()

        """
        if self.dataInput.dataCollectionId is not None:
            # Set ISPyB to running
            self.autoProcIntegrationId, self.autoProcProgramId = \
              EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                         processingCommandLine=self.processingCommandLine,
                                                         processingPrograms=self.processingPrograms,
                                                         isAnom=True,
                                                         timeStart=self.timeStart)
        """
        self.index_plugin.dataInput = xsDataInputCrystFEL
        self.index_plugin.executeSynchronous()

        if self.index_plugin.isFailure():
            self.ERROR('indexamajig: Failed')
            self.setFailure()
            return
        else:
            self.screen('indexamajig: Finished')

        self.process_hkl_plugin.dataInput = xsDataInputCrystFEL
        self.process_hkl_plugin_odd.dataInput = xsDataInputCrystFEL
        self.process_hkl_plugin_even.dataInput = xsDataInputCrystFEL
        self.partialator_plugin.dataInput = xsDataInputCrystFEL
        self.post_process_plugin.dataInput = xsDataInputCrystFEL

        self.process_hkl_plugin_odd.process_hkl_options += " --odd-only"
        self.process_hkl_plugin_even.process_hkl_options += " --even-only"


        self.process_hkl_plugin.executeSynchronous()
        if self.process_hkl_plugin.isFailure():
            self.ERROR('process_hkl: Failed')
            self.setFailure()
            return
        else:
            self.screen('process_hkl: Finished') 


        self.process_hkl_plugin_odd.dataInput.hklFile = XSDataString(self.hklFilename + "_o")
        self.process_hkl_plugin_odd.executeSynchronous()
        if self.process_hkl_plugin_odd.isFailure():
            self.ERROR('process_hkl_odd: Failed')
            self.setFailure()
            return
        else:
            self.screen('process_hkl_odd: Finished')


        self.process_hkl_plugin_even.dataInput.hklFile = XSDataString(self.hklFilename + "_e")
        self.process_hkl_plugin_even.executeSynchronous()
        if self.process_hkl_plugin_even.isFailure():
            self.ERROR('process_hkl_even: Failed')
            self.setFailure()
            return
        else:
            self.screen('process_hkl_even: Finished') 

      
        self.partialator_plugin.executeSynchronous()
        if self.partialator_plugin.isFailure():
            self.ERROR('partialator: Failed')
            self.setFailure()
            return
        else:
            self.screen('partialator: Finished')

        self.post_process_plugin.executeSynchronous()
        if self.post_process_plugin.isFailure():
            self.ERROR('post_process: Failed')
            self.setFailure()
            return
        else:
            self.screen('post_process: Finished')
 
        """ 
        self.edPluginExecCrystFELScale.dataInput = xsDataInputCrystFEL
        self.edPluginExecCrystFELScale.executeSynchronous()

        if self.edPluginExecCrystFELScale.isFailure():
            self.ERROR('CrystFELScalev1_0: Failed')
            self.setFailure()
            return
        else:
            self.screen('CrystFELScalev1_0: Finished')

        self.timeEnd = time.localtime()
        """

        # Upload to ISPyB
        """