def doSuccessMXWaitFile(self, _edPlugin=None):
        self.DEBUG("EDPluginControlID29CreateThumbnailv1_0.doSuccessMXWaitFile")
        self.retrieveSuccessMessages(_edPlugin, "EDPluginControlID29CreateThumbnailv1_0.doSuccessMXWaitFile")
        # Check that the image is really there
        if not self.edPluginMXWaitFile.dataOutput.timedOut.value:
            # Workaround for ESRF lag problem
            if EDUtilsPath.isESRF():
                time.sleep(2)
                if self.isH5:
                    diffractionImagePath = self.dataInput.diffractionImage.path.value
                    imageNumber = EDUtilsImage.getImageNumber(diffractionImagePath)
                    xsDataInputControlH5ToCBF = XSDataInputControlH5ToCBF()
                    xsDataInputControlH5ToCBF.hdf5File = XSDataFile(XSDataString(diffractionImagePath))
                    xsDataInputControlH5ToCBF.imageNumber = XSDataInteger(imageNumber)
                    xsDataInputControlH5ToCBF.hdf5ImageNumber = XSDataInteger(self.h5FileNumber)
                    xsDataInputControlH5ToCBF.ispybDataCollection = None
                    xsDataInputControlH5ToCBF.forcedOutputDirectory = XSDataFile(XSDataString(self.getWorkingDirectory()))
                    edPluginControlH5ToCBF = self.loadPlugin(self.strPluginControlH5ToCBF, "ControlH5ToCBF_{0:04d}".format(imageNumber))
                    edPluginControlH5ToCBF.dataInput = xsDataInputControlH5ToCBF
                    edPluginControlH5ToCBF.executeSynchronous()
                    cbfFile = edPluginControlH5ToCBF.dataOutput.outputCBFFile
                    self.edPluginExecThumbnail.dataInput.image = cbfFile

            # The image is here - make the first thumbnail
            self.edPluginExecThumbnail.connectSUCCESS(self.doSuccessExecThumbnail)
            self.edPluginExecThumbnail.connectFAILURE(self.doFailureExecThumbnail)
            self.edPluginExecThumbnail.executeSynchronous()
        else:
            self.error("Time-out while waiting for image!")
            self.setFailure()
 def finallyProcess(self, _edPlugin=None):
     EDPluginExec.finallyProcess(self, _edPlugin)
     self.DEBUG("EDPluginExecSimpleHTMLPagev1_1.finallyProcess...")
     xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage()
     xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), self.strHtmlFileName))))
     xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(self.getWorkingDirectory())))
     # Write workflowStepReport HTML page
     pathToIndexFile = self.workflowStepReport.renderHtml(self.getWorkingDirectory(), nameOfIndexFile=self.strHtmlFileName)
     pathToJsonFile = self.workflowStepReport.renderJson(self.getWorkingDirectory())
     # Store in Pyarch
     if EDUtilsPath.isESRF() or EDUtilsPath.isEMBL():
         strPyarchPath = None
         if self.xsDataResultCharacterisation is not None:
             strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection())
         if strPyarchPath is None:
             # For debugging purposes
             strPyarchPath = EDUtilsPath.getEdnaUserTempFolder()
         EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath)
         xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(strPyarchPath)))
         if not os.path.exists(strPyarchPath):
             os.makedirs(strPyarchPath, 0o755)
         shutil.copy(pathToJsonFile, strPyarchPath)
         pathToJsonFile = os.path.join(strPyarchPath, os.path.basename(pathToJsonFile))
     # Write json file
     xsDataResultSimpleHTMLPage.pathToJsonFile = XSDataFile(XSDataString(pathToJsonFile))
     self.setDataOutput(xsDataResultSimpleHTMLPage)
Ejemplo n.º 3
0
 def createApplicationWorkingDirectory(self):
     """
     Created the working directory of the application (<date>-<application name>)
     First tries to retrieve the base dir from --basedir option or related parameter from constructor
     Otherwise tries to retrieve it from EDNA_BASE_DIRECTORY environment variable
     Otherwise put the base dir as the current directory
     """
     EDVerbose.DEBUG("EDApplication.createApplicationWorkingDirectory")
     strBaseDirectory = self.getBaseDir()
     strDateTime = time.strftime("%Y%m%d-%H%M%S", time.localtime(time.time()))
     self._strFullApplicationWorkingDirectory = os.path.join(strBaseDirectory, self._strWorkingDir)
     # Check that a folder / file with the same name already exists
     if(os.path.exists(self._strFullApplicationWorkingDirectory) or \
         os.path.exists(self._strFullApplicationWorkingDirectory)):
         # It does exist so we have to modify the name of the working directory
         iIndex = 1
         bContinueFlag = True
         while (bContinueFlag):
             self._strFullApplicationWorkingDirectory = os.path.join(strBaseDirectory,
                                                                                     "%s_%d" % \
                                                                                     (strDateTime, \
                                                                                       iIndex))
             if(os.path.isdir(self._strFullApplicationWorkingDirectory) or \
                 os.path.exists(self._strFullApplicationWorkingDirectory)):
                 iIndex += 1
             else:
                 bContinueFlag = False
     # Add _nobackup suffix if run at the ESRF
     strEdnSite = EDUtilsPath.getEdnaSite()
     if EDUtilsPath.isESRF():
         self._strFullApplicationWorkingDirectory += "_nobackup"
     # Make the directory
     os.mkdir(self._strFullApplicationWorkingDirectory)
     # Change it to be the current working directory
     os.chdir(self._strFullApplicationWorkingDirectory)
Ejemplo n.º 4
0
 def process(self, _edObject=None):
     EDPluginExec.process(self)
     self.DEBUG("EDPluginExecPlotGlev1_1.process")
     for dictPlot in self.listPlot:
         if os.path.dirname(dictPlot["script"]) != self.getWorkingDirectory():
             shutil.copy(dictPlot["script"], self.getWorkingDirectory())
         listDataFiles = dictPlot["data"]
         for strDataFileFullPath in listDataFiles:
             strDataFile = os.path.basename(strDataFileFullPath)
             if not os.path.exists(os.path.join(self.getWorkingDirectory(), strDataFile)):
                 shutil.copy(strDataFileFullPath, self.getWorkingDirectory())
         if EDUtilsPath.isESRF():
             # Force PXSOFT version of gle
             strCommand = "/opt/pxsoft/bin/gle -verbosity 0 -r 150 -d jpg %s" % os.path.basename(dictPlot["script"])
         else:
             strCommand = "gle -verbosity 0 -r 150 -d jpg %s" % os.path.basename(dictPlot["script"])
         # Copied from EDPluginExecProcess
         self.DEBUG(self.getBaseName() + ": Processing")
         timer = threading.Timer(float(self.getTimeOut()), self.kill)
         timer.start()
         try:
             self.__subprocess = EDUtilsPlatform.Popen(shlex.split(str(EDUtilsPlatform.escape(strCommand))),
                                                cwd=self.getWorkingDirectory())
             self.__iPID = self.__subprocess.pid
             self.__strExecutionStatus = str(self.__subprocess.wait())
         except OSError as e:
             strErrorMessage = self.getPluginName() + " : required program gle not installed"
             self.error(strErrorMessage)
             self.setFailure()
         finally:
             timer.cancel()
Ejemplo n.º 5
0
 def finallyProcess(self, _edPlugin=None):
     EDPluginExec.finallyProcess(self, _edPlugin)
     self.DEBUG("EDPluginExecSimpleHTMLPagev1_1.finallyProcess...")
     xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage()
     xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), self.strHtmlFileName))))
     xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(self.getWorkingDirectory())))
     # Write workflowStepReport HTML page
     pathToIndexFile = self.workflowStepReport.renderHtml(self.getWorkingDirectory(), nameOfIndexFile=self.strHtmlFileName)
     pathToJsonFile = self.workflowStepReport.renderJson(self.getWorkingDirectory())
     # Store in Pyarch
     if EDUtilsPath.isESRF() or EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV() or EDUtilsPath.isALBA:
         strPyarchPath = None
         if self.xsDataResultCharacterisation is not None:
             strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection())
         if strPyarchPath is None:
             # For debugging purposes
             strPyarchPath = EDUtilsPath.getEdnaUserTempFolder()
         EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath)
         xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(strPyarchPath)))
         if not os.path.exists(strPyarchPath):
             os.makedirs(strPyarchPath, 0o755)
         shutil.copy(pathToJsonFile, strPyarchPath)
         pathToJsonFile = os.path.join(strPyarchPath, os.path.basename(pathToJsonFile))
     # Write json file
     xsDataResultSimpleHTMLPage.pathToJsonFile = XSDataFile(XSDataString(pathToJsonFile))
     self.setDataOutput(xsDataResultSimpleHTMLPage)
 def sendEmail(self, _strSubject, _strMessage):
     """Sends an email to the EDNA contact person (if configured)."""
     strTime = "%.1f s" % (self.tStop - self.tStart)
     if EDUtilsPath.isESRF():
         strPathImage = None
         for dataSet in self.dataInput.dataSet:
             for imageFile in dataSet.imageFile:
                 strPathImage = imageFile.path.value
                 break
         if strPathImage is not None:
             (strBeamline, strProposal, strPrefix) = self.getBeamlineProposalFromPath(strPathImage)
         else:
             strBeamline = "Unknown"
             strProposal = "Unknown"
             strPrefix = "Unknown"
         strHost = socket.gethostname()
         strSubject = "EDNA ch %s %s %s %s %s (%s)" % (_strSubject, strBeamline, strProposal, strPrefix, strHost, strTime)
     else:
         strSubject = "EDNA %s : %s (%s)" % (_strSubject, EDUtilsPath.getEdnaSite(), strTime)
     self.DEBUG("EDPluginControlInterfaceToMXCuBEv1_4.sendEmail: Subject = %s" % strSubject)
     self.DEBUG("EDPluginControlInterfaceToMXCuBEv1_4.sendEmail: Message:")
     self.DEBUG(_strMessage)
     if self.strEDNAContactEmail == None:
         self.DEBUG("EDPluginControlInterfaceToMXCuBEv1_4.sendEmail: No email address configured!")
     elif not EDUtilsPath.getEdnaSite().startswith("ESRF"):
         self.DEBUG("EDPluginControlInterfaceToMXCuBEv1_4.sendEmail: Not executed at the ESRF! EDNA_SITE=%s" % EDUtilsPath.getEdnaSite())
     else:
         try:
             self.DEBUG("Sending message to %s." % self.strEDNAContactEmail)
             self.DEBUG("Message: %s" % _strMessage)
             strMessage = "EDNA_HOME = %s\n" % EDUtilsPath.getEdnaHome()
             strMessage += "EDNA_SITE = %s\n" % EDUtilsPath.getEdnaSite()
             strMessage += "PLUGIN_NAME = %s\n" % self.getPluginName()
             strMessage += "working_dir = %s\n\n" % self.getWorkingDirectory()
             strMessage += "Reference images:\n"
             xsDataInputMXCuBE = self.getDataInput()
             for xsDataSetMXCuBE in xsDataInputMXCuBE.getDataSet():
                 for xsDataFile in xsDataSetMXCuBE.getImageFile():
                     strMessage += "%s\n" % xsDataFile.getPath().getValue()
             strMessage += "\n"
             strMessage += _strMessage
             strEmailMsg = ("From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s" % (self.strEDNAEmailSender, \
                                                                             self.strEDNAContactEmail, \
                                                                             strSubject, strMessage))
             server = smtplib.SMTP("localhost")
             server.sendmail(self.strEDNAEmailSender, self.strEDNAContactEmail, strEmailMsg)
             server.quit()
         except:
             self.ERROR("Error when sending email message!")
             self.writeErrorTrace()
Ejemplo n.º 7
0
 def sendEmail(self, _strSubject, _strMessage):
     """Sends an email to the EDNA contact person (if configured)."""
     strTime = "%.1f s" % (self.tStop - self.tStart)
     if EDUtilsPath.isESRF():
         strPathImage = None
         for dataSet in self.dataInput.dataSet:
             for imageFile in dataSet.imageFile:
                 strPathImage = imageFile.path.value
                 break
         if strPathImage is not None:
             (strBeamline, strProposal, strPrefix) = self.getBeamlineProposalFromPath(strPathImage)
         else:
             strBeamline = "Unknown"
             strProposal = "Unknown"
             strPrefix = "Unknown"
         strHost = socket.gethostname()
         strSubject = "EDNA ch %s %s %s %s %s (%s)" % (_strSubject, strBeamline, strProposal, strPrefix, strHost, strTime)
     else:
         strSubject = "EDNA %s : %s (%s)" % (_strSubject, EDUtilsPath.getEdnaSite(), strTime)
     self.DEBUG("EDPluginControlInterfaceToMXCuBEv1_3.sendEmail: Subject = %s" % strSubject)
     self.DEBUG("EDPluginControlInterfaceToMXCuBEv1_3.sendEmail: Message:")
     self.DEBUG(_strMessage)
     if self.strEDNAContactEmail == None:
         self.DEBUG("EDPluginControlInterfaceToMXCuBEv1_3.sendEmail: No email address configured!")
     elif not EDUtilsPath.getEdnaSite().startswith("ESRF"):
         self.DEBUG("EDPluginControlInterfaceToMXCuBEv1_3.sendEmail: Not executed at the ESRF! EDNA_SITE=%s" % EDUtilsPath.getEdnaSite())
     else:
         try:
             self.DEBUG("Sending message to %s." % self.strEDNAContactEmail)
             self.DEBUG("Message: %s" % _strMessage)
             strMessage = "EDNA_HOME = %s\n" % EDUtilsPath.getEdnaHome()
             strMessage += "EDNA_SITE = %s\n" % EDUtilsPath.getEdnaSite()
             strMessage += "PLUGIN_NAME = %s\n" % self.getPluginName()
             strMessage += "working_dir = %s\n\n" % self.getWorkingDirectory()
             strMessage += "Reference images:\n"
             xsDataInputMXCuBE = self.getDataInput()
             for xsDataSetMXCuBE in xsDataInputMXCuBE.getDataSet():
                 for xsDataFile in xsDataSetMXCuBE.getImageFile():
                     strMessage += "%s\n" % xsDataFile.getPath().getValue()
             strMessage += "\n"
             strMessage += _strMessage
             strEmailMsg = ("From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s" % (self.strEDNAEmailSender, \
                                                                             self.strEDNAContactEmail, \
                                                                             strSubject, strMessage))
             server = smtplib.SMTP("localhost")
             server.sendmail(self.strEDNAEmailSender, self.strEDNAContactEmail, strEmailMsg)
             server.quit()
         except:
             self.ERROR("Error when sending email message!")
             self.writeErrorTrace()
Ejemplo n.º 8
0
def parse_xds_file(path):
    parsed = dict()
    loaded = _load_xds_file(path)

    # Workaround for problem in ESRF XDS.INP generation during burn strategy workflow execution
    if EDUtilsPath.isESRF():
        has_added_spot_range = False
        tmp_loaded = []
        for kw, args in loaded:
            if kw == "DATA_RANGE=":
                data_range = args
                tmp_loaded.append((kw, args))
            elif kw == "SPOT_RANGE=":
                if len(args) == 2:
                    tmp_loaded.append((kw, args))
                    has_added_spot_range = True
            else:
                tmp_loaded.append((kw, args))
        if not has_added_spot_range:
            tmp_loaded.insert(2, ("SPOT_RANGE=", data_range))
            loaded = tmp_loaded

    # pythonize the arguments
    for kw, args in loaded:
        if kw not in CONFIGURATION_PARSERS:
            # XXX maybe log it
            continue
        parser = CONFIGURATION_PARSERS[kw]

        # print 'parsing', args, 'with', parser, 'for kw', kw

        # XXX maybe catch exc and log them
        parsedargs = parser(args)

        # special case, for repeatable params, we add them as a list
        # or append them if they already exist
        if kw in REPEATABLE_PARAMS:
            # print 'keyword', kw, 'is repeatable'
            if kw in parsed:
                parsed[kw].append(parsedargs)
            else:
                parsed[kw] = [parsedargs]
        else:
            # regular case just assign the thing
            parsed[kw] = parsedargs

    return parsed
Ejemplo n.º 9
0
def parse_xds_file(path):
    parsed = dict()
    loaded = _load_xds_file(path)

    # Workaround for problem in ESRF XDS.INP generation during burn strategy workflow execution
    if EDUtilsPath.isESRF():
        has_added_spot_range = False 
        tmp_loaded = []
        for kw, args in loaded:
            if kw == "DATA_RANGE=":
                data_range = args
                tmp_loaded.append((kw, args))
            elif kw == "SPOT_RANGE=":
                if len(args) == 2:
                    tmp_loaded.append((kw, args))
                    has_added_spot_range = True
            else:
                tmp_loaded.append((kw, args))
        if not has_added_spot_range:
            tmp_loaded.insert(2, ("SPOT_RANGE=", data_range))
            loaded = tmp_loaded

    # pythonize the arguments
    for kw, args in loaded:
        if kw not in CONFIGURATION_PARSERS:
            # XXX maybe log it
            continue
        parser = CONFIGURATION_PARSERS[kw]

        #print 'parsing', args, 'with', parser, 'for kw', kw

        # XXX maybe catch exc and log them
        parsedargs = parser(args)

        # special case, for repeatable params, we add them as a list
        # or append them if they already exist
        if kw in REPEATABLE_PARAMS:
            #print 'keyword', kw, 'is repeatable'
            if kw in parsed:
                parsed[kw].append(parsedargs)
            else:
                parsed[kw] = [parsedargs]
        else:
            # regular case just assign the thing
            parsed[kw] = parsedargs

    return parsed
    def doSuccessMXWaitFile(self, _edPlugin=None):
        self.DEBUG(
            "EDPluginControlID29CreateThumbnailv1_0.doSuccessMXWaitFile")
        self.retrieveSuccessMessages(
            _edPlugin,
            "EDPluginControlID29CreateThumbnailv1_0.doSuccessMXWaitFile")
        # Check that the image is really there
        if not self.edPluginMXWaitFile.dataOutput.timedOut.value:
            # Workaround for ESRF lag problem
            if EDUtilsPath.isESRF():
                time.sleep(2)
                if self.isH5:
                    diffractionImagePath = self.dataInput.diffractionImage.path.value
                    imageNumber = EDUtilsImage.getImageNumber(
                        diffractionImagePath)
                    xsDataInputControlH5ToCBF = XSDataInputControlH5ToCBF()
                    xsDataInputControlH5ToCBF.hdf5File = XSDataFile(
                        XSDataString(diffractionImagePath))
                    xsDataInputControlH5ToCBF.imageNumber = XSDataInteger(
                        imageNumber)
                    xsDataInputControlH5ToCBF.hdf5ImageNumber = XSDataInteger(
                        self.h5FileNumber)
                    xsDataInputControlH5ToCBF.ispybDataCollection = None
                    xsDataInputControlH5ToCBF.forcedOutputDirectory = XSDataFile(
                        XSDataString(self.getWorkingDirectory()))
                    edPluginControlH5ToCBF = self.loadPlugin(
                        self.strPluginControlH5ToCBF,
                        "ControlH5ToCBF_{0:04d}".format(imageNumber))
                    edPluginControlH5ToCBF.dataInput = xsDataInputControlH5ToCBF
                    edPluginControlH5ToCBF.executeSynchronous()
                    cbfFile = edPluginControlH5ToCBF.dataOutput.outputCBFFile
                    self.edPluginExecThumbnail.dataInput.image = cbfFile

            # The image is here - make the first thumbnail
            self.edPluginExecThumbnail.connectSUCCESS(
                self.doSuccessExecThumbnail)
            self.edPluginExecThumbnail.connectFAILURE(
                self.doFailureExecThumbnail)
            self.edPluginExecThumbnail.executeSynchronous()
        else:
            self.error("Time-out while waiting for image!")
            self.setFailure()
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        pathToStartImage = None
        pathToEndImage = None
        userName = os.environ["USER"]
        beamline = "unknown"
        proposal = "unknown"

        # If we have a data collection id, use it
        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            identifier = str(self.dataInput.dataCollectionId.value)
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            self.edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
            directory = ispybDataCollection.imageDirectory
            template = ispybDataCollection.fileTemplate.replace("%04d", "####")
            imageNoStart = ispybDataCollection.startImageNumber
            imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1

#            # DEBUG we set the end image to 20 in order to speed up things
#            self.warning("End image set to 20 (was {0})".format(imageNoEnd))
#            imageNoEnd = 20
            pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
        else:
            identifier = str(int(time.time()))
            directory = self.dataInput.dirN.value
            template = self.dataInput.templateN.value
            imageNoStart = self.dataInput.fromN.value
            imageNoEnd = self.dataInput.toN.value
            fileTemplate = template.replace("####", "%04d")
            pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)

        # Try to get proposal from path
        if EDUtilsPath.isESRF():
            listDirectory = directory.split(os.sep)
            try:
                if listDirectory[1] == "data":
                    if listDirectory[2] == "visitor":
                        beamline = listDirectory[4]
                        proposal = listDirectory[3]
                    else:
                        beamline = listDirectory[2]
                        proposal = listDirectory[4]
            except:
                beamline = "unknown"
                proposal = userName


        if imageNoEnd - imageNoStart < 8:
            error_message = "There are fewer than 8 images, aborting"
            self.addErrorMessage(error_message)
            self.ERROR(error_message)
            self.setFailure()
            return

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        # Make results directory
        self.resultsDirectory = os.path.join(processDirectory, "results")
        if not os.path.exists(self.resultsDirectory):
            os.makedirs(self.resultsDirectory, 0755)

        # Create path to pyarch
        self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
        self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
        if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
            os.makedirs(self.pyarchDirectory, 0755)

        # Determine pyarch prefix
        listPrefix = template.split("_")
        self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])

        isH5 = False
        if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]):
            minSizeFirst = 6000000
            minSizeLast = 6000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
            minSizeFirst = 2000000
            minSizeLast = 2000000
        elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(directory,
                                            self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(directory,
                                          self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True
        else:
            minSizeFirst = 1000000
            minSizeLast = 1000000

        fWaitFileTimeout = 3600  # s

        xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
        xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
        self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
        self.edPluginWaitFileFirst.executeSynchronous()
        if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
            strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
            self.addWarningMessage(strWarningMessage)
            self.WARNING(strWarningMessage)

        xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
        xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
        self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
        self.edPluginWaitFileLast.executeSynchronous()
        if self.edPluginWaitFileLast.dataOutput.timedOut.value:
            strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
            self.addErrorMessage(strErrorMessage)
            self.ERROR(strErrorMessage)
            self.setFailure()



        # Prepare input to execution plugin
        xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
        xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
        if self.doAnomAndNonanom:
            xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
            xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
        if isH5:
            masterFilePath = os.path.join(directory,
                                          self.eiger_template_to_master(template))
            xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(masterFilePath)))
            if self.doAnomAndNonanom:
                xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(masterFilePath)))
        else:
            xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(pathToStartImage)))
            if self.doAnomAndNonanom:
                xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(pathToStartImage)))
        timeStart = time.localtime()
        self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
        self.edPluginExecXia2DIALSAnom.execute()
        if self.doAnomAndNonanom:
            self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
            self.edPluginExecXia2DIALSNoanom.execute()
        self.edPluginExecXia2DIALSAnom.synchronize()
        if self.doAnomAndNonanom:
            self.edPluginExecXia2DIALSNoanom.synchronize()
        timeEnd = time.localtime()

        # Upload to ISPyB
        self.uploadToISPyB(self.edPluginExecXia2DIALSAnom, True, proposal, timeStart, timeEnd)
        if self.doAnomAndNonanom:
            self.uploadToISPyB(self.edPluginExecXia2DIALSNoanom, False, proposal, timeStart, timeEnd)
 def preProcess(self, _edObject=None):
     EDPluginControl.preProcess(self)
     self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess")
     # Check that the input image exists and is of the expected type
     strPathToDiffractionImage = self.dataInput.diffractionImage.path.value
     strImageFileNameExtension = os.path.splitext(
         strPathToDiffractionImage)[1]
     if not strImageFileNameExtension in [
             ".img", ".marccd", ".mccd", ".cbf", ".h5"
     ]:
         self.error(
             "Unknown image file name extension for pyarch thumbnail generator: %s"
             % strPathToDiffractionImage)
         self.setFailure()
     else:
         # Load the MXWaitFile plugin
         xsDataInputMXWaitFile = XSDataInputMXWaitFile()
         pathToImageFile = strPathToDiffractionImage
         # Quite ugly hack to avoid lag problems at the ESRF:
         if EDUtilsPath.isESRF() or EDUtilsPath.isALBA():
             if any(beamline in strPathToDiffractionImage
                    for beamline in ["id23eh1", "id29", "id30b"]):
                 # Pilatus 6M
                 self.minImageSize = 6000000
             elif any(beamline in strPathToDiffractionImage
                      for beamline in ["id23eh2", "id30a1"]):
                 # Pilatus3 2M
                 self.minImageSize = 2000000
             elif strImageFileNameExtension == ".h5":
                 self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath(
                     pathToImageFile)
                 pathToImageFile = self.h5DataFilePath
                 self.isH5 = True
         elif EDUtilsPath.isEMBL():
             self.minImageSize = 10000
         xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
         xsDataInputMXWaitFile.setFile(
             XSDataFile(XSDataString(pathToImageFile)))
         if self.getDataInput().getWaitForFileTimeOut():
             xsDataInputMXWaitFile.setTimeOut(
                 self.getDataInput().getWaitForFileTimeOut())
         self.edPluginMXWaitFile = self.loadPlugin(
             self.strMXWaitFilePluginName)
         self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
         # Load the execution plugin
         self.edPluginExecThumbnail = self.loadPlugin(
             self.strExecThumbnailPluginName)
         xsDataInputMXThumbnail = XSDataInputMXThumbnail()
         xsDataInputMXThumbnail.image = self.getDataInput(
         ).getDiffractionImage()
         xsDataInputMXThumbnail.height = XSDataInteger(1024)
         xsDataInputMXThumbnail.width = XSDataInteger(1024)
         xsDataInputMXThumbnail.format = self.dataInput.format
         # Output path
         strImageNameWithoutExt = os.path.basename(
             os.path.splitext(strPathToDiffractionImage)[0])
         strImageDirname = os.path.dirname(strPathToDiffractionImage)
         if self.getDataInput().getForcedOutputDirectory():
             strForcedOutputDirectory = self.getDataInput(
             ).getForcedOutputDirectory().getPath().getValue()
             if not os.access(strForcedOutputDirectory, os.W_OK):
                 self.error("Cannot write to forced output directory : %s" %
                            strForcedOutputDirectory)
                 self.setFailure()
             else:
                 self.strOutputPathWithoutExtension = os.path.join(
                     strForcedOutputDirectory, strImageNameWithoutExt)
         else:
             # Try to store in the ESRF pyarch directory
             strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
                 strImageDirname)
             # Check that output pyarch path exists and is writeable:
             bIsOk = False
             if strOutputDirname:
                 if not os.path.exists(strOutputDirname):
                     # Try to create the directory
                     try:
                         os.makedirs(strOutputDirname)
                         bIsOk = True
                     except Exception as e:
                         self.WARNING("Couldn't create the directory %s" %
                                      strOutputDirname)
                 elif os.access(strOutputDirname, os.W_OK):
                     bIsOk = True
             if not bIsOk:
                 self.warning("Cannot write to pyarch directory: %s" %
                              strOutputDirname)
                 strTmpUser = os.path.join("/tmp", os.environ["USER"])
                 if not os.path.exists(strTmpUser):
                     os.mkdir(strTmpUser, 0o755)
                 strOutputDirname = tempfile.mkdtemp(
                     prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser)
                 os.chmod(strOutputDirname, 0o755)
                 self.warning("Writing thumbnail images to: %s" %
                              strOutputDirname)
             self.strOutputPathWithoutExtension = os.path.join(
                 strOutputDirname, strImageNameWithoutExt)
         if self.dataInput.format is not None:
             self.strSuffix = self.dataInput.format.value.lower()
             self.strImageFormat = self.dataInput.format.value.upper()
         self.strOutputPath = os.path.join(
             self.strOutputPathWithoutExtension + "." + self.strSuffix)
         xsDataInputMXThumbnail.setOutputPath(
             XSDataFile(XSDataString(self.strOutputPath)))
         self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
Ejemplo n.º 13
0
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlXDSAPPv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        pathToStartImage = None
        pathToEndImage = None
        userName = os.environ["USER"]
        beamline = "unknown"
        proposal = "unknown"

        if self.dataInput.startImageNumber is not None:
            imageNoStart = self.dataInput.startImageNumber.value
        if self.dataInput.endImageNumber is not None:
            imageNoEnd = self.dataInput.endImageNumber.value
        # If we have a data collection id, use it
        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            identifier = str(self.dataInput.dataCollectionId.value)
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            self.edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
            directory = ispybDataCollection.imageDirectory
            template = ispybDataCollection.fileTemplate.replace("%04d", "####")
            if imageNoStart is None:
                imageNoStart = ispybDataCollection.startImageNumber
            if imageNoEnd is None:
                imageNoEnd = ispybDataCollection.startImageNumber + \
                             ispybDataCollection.numberOfImages - 1

#            # DEBUG we set the end image to 20 in order to speed up things
#            self.warning("End image set to 20 (was {0})".format(imageNoEnd))
#            imageNoEnd = 20
            pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
        else:
            identifier = str(int(time.time()))
            directory = self.dataInput.dirN.value
            template = self.dataInput.templateN.value
            fileTemplate = template.replace("####", "%04d")
            pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)

        # Try to get proposal from path
        if EDUtilsPath.isESRF():
            listDirectory = directory.split(os.sep)
            try:
                if listDirectory[1] == "data":
                    if listDirectory[2] == "visitor":
                        beamline = listDirectory[4]
                        proposal = listDirectory[3]
                    else:
                        beamline = listDirectory[2]
                        proposal = listDirectory[4]
            except:
                beamline = "unknown"
                proposal = userName


        if imageNoEnd - imageNoStart < 8:
            error_message = "There are fewer than 8 images, aborting"
            self.addErrorMessage(error_message)
            self.ERROR(error_message)
            self.setFailure()
            return

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        # Make results directory
        self.resultsDirectory = os.path.join(processDirectory, "results")
        if not os.path.exists(self.resultsDirectory):
            os.makedirs(self.resultsDirectory, 0o755)

        # Create path to pyarch
        if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
                "XDSAPP", self.dataInput.dataCollectionId.value)
        else:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
        self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
        if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
            try:
                os.makedirs(self.pyarchDirectory, 0o755)
            except:
                self.pyarchDirectory = None

        # Determine pyarch prefix
        listPrefix = template.split("_")
        self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])

        isH5 = False
        if any(beamline in pathToStartImage for beamline in ["id30b"]):
            minSizeFirst = 6000000
            minSizeLast = 6000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
            minSizeFirst = 2000000
            minSizeLast = 2000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(directory,
                                            self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(directory,
                                          self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True
        else:
            minSizeFirst = 1000000
            minSizeLast = 1000000

        fWaitFileTimeout = 3600  # s

        xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
        xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
        self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
        self.edPluginWaitFileFirst.executeSynchronous()
        if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
            strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
            self.addWarningMessage(strWarningMessage)
            self.WARNING(strWarningMessage)

        xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
        xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
        self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
        self.edPluginWaitFileLast.executeSynchronous()
        if self.edPluginWaitFileLast.dataOutput.timedOut.value:
            strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
            self.addErrorMessage(strErrorMessage)
            self.ERROR(strErrorMessage)
            self.setFailure()


        if self.dataInput.hdf5ToCbfDirectory is not None:
            dir = self.dataInput.hdf5ToCbfDirectory.path.value
            pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0]

        self.timeStart = time.localtime()
        # Prepare input to execution plugin
        if self.doAnom:
            xsDataInputXDSAPPAnom = XSDataInputXDSAPP()
            xsDataInputXDSAPPAnom.startImageNumber = self.dataInput.startImageNumber
            xsDataInputXDSAPPAnom.endImageNumber = self.dataInput.endImageNumber
            xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True)
            xsDataInputXDSAPPAnom.image = XSDataFile(XSDataString(pathToStartImage))
            if self.xdsAppSpacegroup is not None:
                xsDataInputXDSAPPAnom.spacegroup = XSDataString(self.xdsAppSpacegroup)
            self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom
            self.edPluginExecXDSAPPAnom.execute()
            if self.dataInput.dataCollectionId is not None:
                # Set ISPyB to started
                self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingPrograms=self.processingPrograms, isAnom=True,
                                                             timeStart=self.timeStart)
        if self.doNoanom:
            xsDataInputXDSAPPNoanom = XSDataInputXDSAPP()
            xsDataInputXDSAPPNoanom.startImageNumber = self.dataInput.startImageNumber
            xsDataInputXDSAPPNoanom.endImageNumber = self.dataInput.endImageNumber
            xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False)
            xsDataInputXDSAPPNoanom.image = XSDataFile(XSDataString(pathToStartImage))
            if self.xdsAppSpacegroup is not None:
                xsDataInputXDSAPPNoanom.spacegroup = XSDataString(self.xdsAppSpacegroup)
            self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom
            self.edPluginExecXDSAPPNoanom.execute()
            if self.dataInput.dataCollectionId is not None:
                # Set ISPyB to started
                self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
                    EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                               processingPrograms=self.processingPrograms, isAnom=False,
                                                               timeStart=self.timeStart)
        if self.doAnom:
            self.edPluginExecXDSAPPAnom.synchronize()
            xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput
            # Run XSCALE even if XSCALE.LP is present
            strPathXscaleLpAnom = self.runXscale(self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True)
        if self.doNoanom:
            self.edPluginExecXDSAPPNoanom.synchronize()
            xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput
            strPathXscaleLpNoanom = self.runXscale(self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True)
        self.timeEnd = time.localtime()
        # Upload to ISPyB
        if self.dataInput.dataCollectionId is not None:
            # Check if we should use XDS_ASCII_to_XML.pl
            if self.doAnom:
                if self.useXdsAsciiToXml:
                    # Only for anom runs
                    self.runXdsAsciiToXml(xsDataResultXDSAPPAnom,
                                          self.dataInput.dataCollectionId.value,
                                          self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
                else:
                    self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPAnom, processDirectory, template,
                                       strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd,
                                       self.dataInput.dataCollectionId.value,
                                       self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
                    if self.hasUploadedAnomResultsToISPyB:
                        self.screen("Anom results uploaded to ISPyB")
                    else:
                        self.ERROR("Could not upload anom results to ISPyB!")

            if self.doNoanom:
                self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPNoanom, processDirectory, template,
                                   strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd,
                                   self.dataInput.dataCollectionId.value,
                                   self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom)
                if self.hasUploadedNoanomResultsToISPyB:
                    self.screen("Noanom results uploaded to ISPyB")
                else:
                    self.ERROR("Could not upload noanom results to ISPyB!")
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        pathToStartImage = None
        pathToEndImage = None
        userName = os.environ["USER"]
        beamline = "unknown"
        proposal = "unknown"

        # If we have a data collection id, use it
        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
            )
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            self.edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
            directory = ispybDataCollection.imageDirectory
            if EDUtilsPath.isEMBL():
                template = ispybDataCollection.fileTemplate.replace(
                    "%05d", "####")
            else:
                template = ispybDataCollection.fileTemplate.replace(
                    "%04d", "####")
            if self.dataInput.startFrame is None:
                imageNoStart = ispybDataCollection.startImageNumber
            else:
                imageNoStart = self.dataInput.startFrame.value
            if self.dataInput.endFrame is None:
                imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
            else:
                imageNoEnd = self.dataInput.endFrame.value

#            # DEBUG we set the end image to 20 in order to speed up things
#            self.warning("End image set to 20 (was {0})".format(imageNoEnd))
#            imageNoEnd = 20
            pathToStartImage = os.path.join(
                directory, ispybDataCollection.fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(
                directory, ispybDataCollection.fileTemplate % imageNoEnd)
#        else:
#            directory = self.dataInput.dirN.value
#            template = self.dataInput.templateN.value
#            imageNoStart = self.dataInput.fromN.value
#            imageNoEnd = self.dataInput.toN.value
#            fileTemplate = template.replace("####", "%04d")
#            pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
#            pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)

# Try to get proposal from path
        if EDUtilsPath.isESRF():
            listDirectory = directory.split(os.sep)
            try:
                if listDirectory[1] == "data":
                    if listDirectory[2] == "visitor":
                        beamline = listDirectory[4]
                        proposal = listDirectory[3]
                    else:
                        beamline = listDirectory[2]
                        proposal = listDirectory[4]
            except:
                beamline = "unknown"
                proposal = userName

        if imageNoEnd - imageNoStart < 8:
            error_message = "There are fewer than 8 images, aborting"
            self.addErrorMessage(error_message)
            self.ERROR(error_message)
            self.setFailure()
            return

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        # Make results directory
        self.resultsDirectory = os.path.join(processDirectory, "results")
        if not os.path.exists(self.resultsDirectory):
            os.makedirs(self.resultsDirectory, 0o755)

        # Create path to pyarch
        if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
                beamline, "XIA2_DIALS", self.dataInput.dataCollectionId.value)
        else:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
                self.resultsDirectory)
        if self.pyarchDirectory is not None:
            self.pyarchDirectory = self.pyarchDirectory.replace(
                'PROCESSED_DATA', 'RAW_DATA')
            if not os.path.exists(self.pyarchDirectory):
                try:
                    os.makedirs(self.pyarchDirectory, 0o755)
                except:
                    self.pyarchDirectory = None

        # Determine pyarch prefix
        listPrefix = template.split("_")
        self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3],
                                                   listPrefix[-2])

        isH5 = False
        if any(beamline in pathToStartImage
               for beamline in ["id23eh1", "id29"]):
            minSizeFirst = 6000000
            minSizeLast = 6000000
        elif any(beamline in pathToStartImage
                 for beamline in ["id23eh2", "id30a1"]):
            minSizeFirst = 2000000
            minSizeLast = 2000000
        elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(
                directory,
                self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(
                directory, self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True
        else:
            minSizeFirst = 1000000
            minSizeLast = 1000000

        if EDUtilsPath.isEMBL():
            fWaitFileTimeout = 60
        else:
            fWaitFileTimeout = 3600  # s

        xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileFirst.file = XSDataFile(
            XSDataString(pathToStartImage))
        xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
        self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
        self.edPluginWaitFileFirst.executeSynchronous()
        if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
            strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
                fWaitFileTimeout, pathToStartImage)
            self.addWarningMessage(strWarningMessage)
            self.WARNING(strWarningMessage)

        xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileLast.file = XSDataFile(
            XSDataString(pathToEndImage))
        xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
        self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
        self.edPluginWaitFileLast.executeSynchronous()
        if self.edPluginWaitFileLast.dataOutput.timedOut.value:
            strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (
                fWaitFileTimeout, pathToEndImage)
            self.addErrorMessage(strErrorMessage)
            self.ERROR(strErrorMessage)
            self.setFailure()

        # Prepare input to execution plugin
        if self.doAnom:
            xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
            xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
            xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup
            xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell
            if imageNoStart is not None:
                xsDataInputXia2DIALSAnom.startFrame = XSDataInteger(
                    imageNoStart)
            if imageNoEnd is not None:
                xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd)
        if self.doNoanom:
            xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
            xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
            xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup
            xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell
            if imageNoStart is not None:
                xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger(
                    imageNoStart)
            if imageNoEnd is not None:
                xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd)
        if isH5:
            masterFilePath = os.path.join(
                directory, self.eiger_template_to_master(template))
            if self.doAnom:
                xsDataInputXia2DIALSAnom.addImage(
                    XSDataFile(XSDataString(masterFilePath)))
            if self.doNoanom:
                xsDataInputXia2DIALSNoanom.addImage(
                    XSDataFile(XSDataString(masterFilePath)))
        else:
            if self.doAnom:
                xsDataInputXia2DIALSAnom.addImage(
                    XSDataFile(XSDataString(pathToStartImage)))
            if self.doNoanom:
                xsDataInputXia2DIALSNoanom.addImage(
                    XSDataFile(XSDataString(pathToStartImage)))
        self.timeStart = time.localtime()

        if self.dataInput.dataCollectionId is not None:
            # Set ISPyB to running
            if self.doAnom:
                self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingPrograms,
                                                             isAnom=True,
                                                             timeStart=self.timeStart)
            if self.doNoanom:
                self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingPrograms,
                                                             isAnom=False,
                                                             timeStart=self.timeStart)

        if self.doAnom:
            self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
            self.edPluginExecXia2DIALSAnom.execute()
        if self.doNoanom:
            self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
            self.edPluginExecXia2DIALSNoanom.execute()
        if self.doAnom:
            self.edPluginExecXia2DIALSAnom.synchronize()
        if self.doNoanom:
            self.edPluginExecXia2DIALSNoanom.synchronize()
        self.timeEnd = time.localtime()

        # Upload to ISPyB
        if self.doAnom:
            self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(
                self.edPluginExecXia2DIALSAnom, True, proposal,
                self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom)
            if self.hasUploadedAnomResultsToISPyB:
                self.screen("Anom results uploaded to ISPyB")
            else:
                self.ERROR("Could not upload anom results to ISPyB!")
        if self.doNoanom:
            self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(
                self.edPluginExecXia2DIALSNoanom, False, proposal,
                self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom)
            if self.hasUploadedNoanomResultsToISPyB:
                self.screen("Noanom results uploaded to ISPyB")
            else:
                self.ERROR("Could not upload noanom results to ISPyB!")
Ejemplo n.º 15
0
    def postProcess(self, _edObject=None):
        EDPluginControl.postProcess(self)
        self.DEBUG("EDPluginControlDozorv1_0.postProcess")
        # Write a file to be used with ISPyB or GNUPLOT only if data collection id in input
        dataCollectionId = None
        if self.dataInput.dataCollectionId is None and len(self.dataInput.image) > 0:
            # Only try to obtain data collection id if at the ESRF and path starts with "/data"
            if EDUtilsPath.isESRF() and self.dataInput.image[0].path.value.startswith("/data"):
                xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
                xsDataInputRetrieveDataCollection.image = XSDataImage(self.dataInput.image[0].path)
                edPluginISPyBRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
                edPluginISPyBRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
                edPluginISPyBRetrieveDataCollection.executeSynchronous()
                xsDataResultRetrieveDataCollection = edPluginISPyBRetrieveDataCollection.dataOutput
                if xsDataResultRetrieveDataCollection is not None:
                    dataCollection = xsDataResultRetrieveDataCollection.dataCollection
                    if dataCollection is not None:
                        dataCollectionId = dataCollection.dataCollectionId
        elif self.dataInput.dataCollectionId is not None:
            dataCollectionId = self.dataInput.dataCollectionId.value

        if dataCollectionId is not None:
            minImageNumber = None
            maxImageNumber = None
            minAngle = None
            maxAngle = None
            minDozorValue = None
            maxDozorValue = None
            minResolution = None
            maxResolution = None
            dozorPlotFileName = "dozor_{0}.png".format(dataCollectionId)
            dozorCsvFileName = "dozor_{0}.csv".format(dataCollectionId)
            with open(os.path.join(self.getWorkingDirectory(), dozorCsvFileName), "w") as gnuplotFile:
                gnuplotFile.write("# Data directory: {0}\n".format(self.directory))
                gnuplotFile.write("# File template: {0}\n".format(self.template.replace("%04d", "####")))
                gnuplotFile.write("# {0:>9s}{1:>16s}{2:>16s}{3:>16s}{4:>16s}{5:>16s}\n".format("'Image no'",
                                                                               "'Angle'",
                                                                               "'No of spots'",
                                                                               "'Main score (*10)'",
                                                                               "'Spot score'",
                                                                               "'Visible res.'",
                                                               ))
                for imageDozor in self.dataOutput.imageDozor:
                    gnuplotFile.write("{0:10d},{1:15.3f},{2:15d},{3:15.3f},{4:15.3f},{5:15.3f}\n".format(imageDozor.number.value,
                                                                                                       imageDozor.angle.value,
                                                                                                       imageDozor.spotsNumOf.value,
                                                                                                       10 * imageDozor.mainScore.value,
                                                                                                       imageDozor.spotScore.value,
                                                                                                       imageDozor.visibleResolution.value,
                                                                                                       ))
                    if minImageNumber is None or minImageNumber > imageDozor.number.value:
                        minImageNumber = imageDozor.number.value
                        minAngle = imageDozor.angle.value
                    if maxImageNumber is None or maxImageNumber < imageDozor.number.value:
                        maxImageNumber = imageDozor.number.value
                        maxAngle = imageDozor.angle.value
                    if minDozorValue is None or minDozorValue > imageDozor.mainScore.value:
                        minDozorValue = imageDozor.spotScore.value
                    if maxDozorValue is None or maxDozorValue < imageDozor.mainScore.value:
                        maxDozorValue = imageDozor.spotScore.value

                    # Min resolution: the higher the value the lower the resolution
                    if minResolution is None or minResolution < imageDozor.visibleResolution.value:
                        # Disregard resolution worse than 10.0
                        if imageDozor.visibleResolution.value < 10.0:
                            minResolution = imageDozor.visibleResolution.value

                    # Max resolution: the lower the number the better the resolution
                    if maxResolution is None or maxResolution > imageDozor.visibleResolution.value:
                        maxResolution = imageDozor.visibleResolution.value

            xtics = ""
            if minImageNumber is not None and minImageNumber == maxImageNumber:
                minAngle -= 1.0
                maxAngle += 1.0
            noImages = maxImageNumber - minImageNumber + 1
            if noImages <= 4:
                minImageNumber -= 0.1
                maxImageNumber += 0.1
                deltaAngle = maxAngle - minAngle
                minAngle -= deltaAngle * 0.1 / noImages
                maxAngle += deltaAngle * 0.1 / noImages
                xtics = "1"

            if maxResolution is None or maxResolution > 0.8:
                maxResolution = 0.8
            else:
                maxResolution = int(maxResolution * 10.0) / 10.0

            if minResolution is None or minResolution < 4.5:
                minResolution = 4.5
            else:
                minResolution = int(minResolution * 10.0) / 10.0 + 1

            if maxDozorValue < 0.001 and minDozorValue < 0.001:
                yscale = "set yrange [-0.5:0.5]\n    set ytics 1"
            else:
                yscale = "set autoscale  y"

            gnuplotFile.close()
            gnuplotScript = \
    """#
    set terminal png
    set output '{dozorPlotFileName}'
    set title '{title}'
    set grid x2 y2
    set xlabel 'Image number'
    set x2label 'Angle (degrees)'
    set y2label 'Resolution (A)'
    set ylabel 'Number of spots / Dozor score (*10)'
    set xtics {xtics} nomirror
    set x2tics 
    set ytics nomirror
    set y2tics
    set xrange [{minImageNumber}:{maxImageNumber}]
    set x2range [{minAngle}:{maxAngle}]
    {yscale}
    set y2range [{minResolution}:{maxResolution}]
    set key below
    plot '{dozorCsvFileName}' using 1:3 title 'Number of spots' axes x1y1 with points linetype rgb 'goldenrod' pointtype 7 pointsize 1.5, \
         '{dozorCsvFileName}' using 1:4 title 'Dozor score' axes x1y1 with points linetype 3 pointtype 7 pointsize 1.5, \
         '{dozorCsvFileName}' using 1:6 title 'Visible resolution' axes x1y2 with points linetype 1 pointtype 7 pointsize 1.5
    """.format(title=self.template.replace("%04d", "####"),
               dozorPlotFileName=dozorPlotFileName,
               dozorCsvFileName=dozorCsvFileName,
               minImageNumber=minImageNumber,
               maxImageNumber=maxImageNumber,
               minAngle=minAngle,
               maxAngle=maxAngle,
               minResolution=minResolution,
               maxResolution=maxResolution,
               xtics=xtics,
               yscale=yscale,
               )
            pathGnuplotScript = os.path.join(self.getWorkingDirectory(), "gnuplot.sh")
            data_file = open(pathGnuplotScript, "w")
            data_file.write(gnuplotScript)
            data_file.close()
            oldCwd = os.getcwd()
            os.chdir(self.getWorkingDirectory())
            os.system("{0} {1}".format(self.gnuplot, pathGnuplotScript))
            os.chdir(oldCwd)

            self.dataOutput.dozorPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), dozorPlotFileName)))

            if self.dataInput.processDirectory is not None:
                processDirectory = self.dataInput.processDirectory.path.value
            else:
                processDirectory = os.path.join(self.directory.replace("RAW_DATA", "PROCESSED_DATA"), "DozorPlot")
            resultsDirectory = os.path.join(processDirectory, "results")
            dozorPlotResultPath = os.path.join(resultsDirectory, dozorPlotFileName)
            dozorCsvResultPath = os.path.join(resultsDirectory, dozorCsvFileName)
            try:
                if not os.path.exists(resultsDirectory):
                    os.makedirs(resultsDirectory, 0o755)
                shutil.copy(os.path.join(self.getWorkingDirectory(), dozorPlotFileName), dozorPlotResultPath)
                shutil.copy(os.path.join(self.getWorkingDirectory(), dozorCsvFileName), dozorCsvResultPath)
            except:
                self.warning("Couldn't copy files to results directory: {0}".format(resultsDirectory))

            try:
                # Create paths on pyarch
                dozorPlotPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(dozorPlotResultPath)
                dozorCsvPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(dozorCsvResultPath)
                if not os.path.exists(os.path.dirname(dozorPlotPyarchPath)):
                    os.makedirs(os.path.dirname(dozorPlotPyarchPath), 0o755)
                shutil.copy(dozorPlotResultPath, dozorPlotPyarchPath)
                shutil.copy(dozorCsvResultPath, dozorCsvPyarchPath)
                # Upload to data collection
                xsDataInputISPyBSetImageQualityIndicatorsPlot = XSDataInputISPyBSetImageQualityIndicatorsPlot()
                xsDataInputISPyBSetImageQualityIndicatorsPlot.dataCollectionId = XSDataInteger(dataCollectionId)
                xsDataInputISPyBSetImageQualityIndicatorsPlot.imageQualityIndicatorsPlotPath = XSDataString(dozorPlotPyarchPath)
                xsDataInputISPyBSetImageQualityIndicatorsPlot.imageQualityIndicatorsCSVPath = XSDataString(dozorCsvPyarchPath)
                EDPluginISPyBSetImageQualityIndicatorsPlot = self.loadPlugin("EDPluginISPyBSetImageQualityIndicatorsPlotv1_4")
                EDPluginISPyBSetImageQualityIndicatorsPlot.dataInput = xsDataInputISPyBSetImageQualityIndicatorsPlot
                EDPluginISPyBSetImageQualityIndicatorsPlot.executeSynchronous()
            except:
                self.warning("Couldn't copy files to pyarch: {0}".format(dozorPlotPyarchPath))

        self.sendMessageToMXCuBE("Processing finished", "info")
        self.setStatusToMXCuBE("Success")
 def preProcess(self, _edObject=None):
     EDPluginControl.preProcess(self)
     self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess")
     # Check that the input image exists and is of the expected type
     strPathToDiffractionImage = self.dataInput.diffractionImage.path.value
     strImageFileNameExtension = os.path.splitext(strPathToDiffractionImage)[1]
     if not strImageFileNameExtension in [".img", ".marccd", ".mccd", ".cbf", ".h5"]:
         self.error("Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage)
         self.setFailure()
     else:
         # Load the MXWaitFile plugin
         xsDataInputMXWaitFile = XSDataInputMXWaitFile()
         pathToImageFile = strPathToDiffractionImage
         # Quite ugly hack to avoid lag problems at the ESRF:
         if EDUtilsPath.isESRF():
             if any(beamline in strPathToDiffractionImage for beamline in ["id23eh1", "id29", "id30b"]):
                 # Pilatus 6M
                 self.minImageSize = 6000000
             elif any(beamline in strPathToDiffractionImage for beamline in ["id23eh2", "id30a1"]):
                 # Pilatus3 2M
                 self.minImageSize = 2000000
             elif strImageFileNameExtension == ".h5":
                 self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath(pathToImageFile)
                 pathToImageFile = self.h5DataFilePath
                 self.isH5 = True
         elif EDUtilsPath.isEMBL():
                 self.minImageSize = 10000
         xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
         xsDataInputMXWaitFile.setFile(XSDataFile(XSDataString(pathToImageFile)))
         if self.getDataInput().getWaitForFileTimeOut():
             xsDataInputMXWaitFile.setTimeOut(self.getDataInput().getWaitForFileTimeOut())
         self.edPluginMXWaitFile = self.loadPlugin(self.strMXWaitFilePluginName)
         self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
         # Load the execution plugin
         self.edPluginExecThumbnail = self.loadPlugin(self.strExecThumbnailPluginName)
         xsDataInputMXThumbnail = XSDataInputMXThumbnail()
         xsDataInputMXThumbnail.image = self.getDataInput().getDiffractionImage()
         xsDataInputMXThumbnail.height = XSDataInteger(1024)
         xsDataInputMXThumbnail.width = XSDataInteger(1024)
         xsDataInputMXThumbnail.format = self.dataInput.format
         # Output path
         strImageNameWithoutExt = os.path.basename(os.path.splitext(strPathToDiffractionImage)[0])
         strImageDirname = os.path.dirname(strPathToDiffractionImage)
         if self.getDataInput().getForcedOutputDirectory():
             strForcedOutputDirectory = self.getDataInput().getForcedOutputDirectory().getPath().getValue()
             if not os.access(strForcedOutputDirectory, os.W_OK):
                 self.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory)
                 self.setFailure()
             else:
                 self.strOutputPathWithoutExtension = os.path.join(strForcedOutputDirectory, strImageNameWithoutExt)
         else:
             # Try to store in the ESRF pyarch directory
             strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strImageDirname)
             # Check that output pyarch path exists and is writeable:
             bIsOk = False
             if strOutputDirname:
                 if not os.path.exists(strOutputDirname):
                     # Try to create the directory
                     try:
                         os.makedirs(strOutputDirname)
                         bIsOk = True
                     except Exception as e:
                         self.WARNING("Couldn't create the directory %s" % strOutputDirname)
                 elif os.access(strOutputDirname, os.W_OK):
                     bIsOk = True
             if not bIsOk:
                 self.warning("Cannot write to pyarch directory: %s" % strOutputDirname)
                 strTmpUser = os.path.join("/tmp", os.environ["USER"])
                 if not os.path.exists(strTmpUser):
                     os.mkdir(strTmpUser, 0755)
                 strOutputDirname = tempfile.mkdtemp(prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser)
                 os.chmod(strOutputDirname, 0755)
                 self.warning("Writing thumbnail images to: %s" % strOutputDirname)
             self.strOutputPathWithoutExtension = os.path.join(strOutputDirname, strImageNameWithoutExt)
         if self.dataInput.format is not None:
             self.strSuffix = self.dataInput.format.value.lower()
             self.strImageFormat = self.dataInput.format.value.upper()
         self.strOutputPath = os.path.join(self.strOutputPathWithoutExtension + "." + self.strSuffix)
         xsDataInputMXThumbnail.setOutputPath(XSDataFile(XSDataString(self.strOutputPath)))
         self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlXDSAPPv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        pathToStartImage = None
        pathToEndImage = None
        userName = os.environ["USER"]
        beamline = "unknown"
        proposal = "unknown"

        # If we have a data collection id, use it
        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            identifier = str(self.dataInput.dataCollectionId.value)
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            self.edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
            directory = ispybDataCollection.imageDirectory
            template = ispybDataCollection.fileTemplate.replace("%04d", "####")
            imageNoStart = ispybDataCollection.startImageNumber
            imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1

#            # DEBUG we set the end image to 20 in order to speed up things
#            self.warning("End image set to 20 (was {0})".format(imageNoEnd))
#            imageNoEnd = 20
            pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
        else:
            identifier = str(int(time.time()))
            directory = self.dataInput.dirN.value
            template = self.dataInput.templateN.value
            imageNoStart = self.dataInput.fromN.value
            imageNoEnd = self.dataInput.toN.value
            fileTemplate = template.replace("####", "%04d")
            pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)

        # Try to get proposal from path
        if EDUtilsPath.isESRF():
            listDirectory = directory.split(os.sep)
            try:
                if listDirectory[1] == "data":
                    if listDirectory[2] == "visitor":
                        beamline = listDirectory[4]
                        proposal = listDirectory[3]
                    else:
                        beamline = listDirectory[2]
                        proposal = listDirectory[4]
            except:
                beamline = "unknown"
                proposal = userName


        if imageNoEnd - imageNoStart < 8:
            error_message = "There are fewer than 8 images, aborting"
            self.addErrorMessage(error_message)
            self.ERROR(error_message)
            self.setFailure()
            return

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        # Make results directory
        self.resultsDirectory = os.path.join(processDirectory, "results")
        if not os.path.exists(self.resultsDirectory):
            os.makedirs(self.resultsDirectory, 0o755)

        isH5 = False
        if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]):
            minSizeFirst = 6000000
            minSizeLast = 6000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
            minSizeFirst = 2000000
            minSizeLast = 2000000
        elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(directory,
                                            self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(directory,
                                          self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True
        else:
            minSizeFirst = 1000000
            minSizeLast = 1000000

        fWaitFileTimeout = 3600  # s

        xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
        xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
        self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
        self.edPluginWaitFileFirst.executeSynchronous()
        if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
            strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
            self.addWarningMessage(strWarningMessage)
            self.WARNING(strWarningMessage)

        xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
        xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
        self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
        self.edPluginWaitFileLast.executeSynchronous()
        if self.edPluginWaitFileLast.dataOutput.timedOut.value:
            strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
            self.addErrorMessage(strErrorMessage)
            self.ERROR(strErrorMessage)
            self.setFailure()


        if self.dataInput.hdf5ToCbfDirectory is not None:
            dir = self.dataInput.hdf5ToCbfDirectory.path.value
            pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0]

        # Prepare input to execution plugin
        xsDataInputXDSAPPAnom = XSDataInputXDSAPP()
        xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True)
        xsDataInputXDSAPPAnom.image = XSDataFile(XSDataString(pathToStartImage))
        if self.xdsAppSpacegroup is not None:
            xsDataInputXDSAPPAnom.spacegroup = XSDataString(self.xdsAppSpacegroup)
        if self.doAnomAndNonanom:
            xsDataInputXDSAPPNoanom = XSDataInputXDSAPP()
            xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False)
            xsDataInputXDSAPPNoanom.image = XSDataFile(XSDataString(pathToStartImage))
            if self.xdsAppSpacegroup is not None:
                xsDataInputXDSAPPNoanom.spacegroup = XSDataString(self.xdsAppSpacegroup)
#        if isH5:
#            masterFilePath = os.path.join(directory,
#                                          self.eiger_template_to_master(template))
#            xsDataInputXDSAPPAnom.addImage(XSDataFile(XSDataString(masterFilePath)))
#            if self.doAnomAndNonanom:
#                xsDataInputXDSAPPNoanom.addImage(XSDataFile(XSDataString(masterFilePath)))
#        else:
#            xsDataInputXDSAPPAnom.addImage(XSDataFile(XSDataString(pathToStartImage)))
#            if self.doAnomAndNonanom:
#                xsDataInputXDSAPPNoanom.addImage(XSDataFile(XSDataString(pathToStartImage)))
        self.timeStart = time.localtime()
        self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom
        self.edPluginExecXDSAPPAnom.execute()
        if self.dataInput.dataCollectionId is not None:
            # Set ISPyB to started
            self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
              EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                         processingPrograms=self.processingPrograms, isAnom=True,
                                                         timeStart=self.timeStart)

        if self.doAnomAndNonanom:
            self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom
            self.edPluginExecXDSAPPNoanom.execute()
            if self.dataInput.dataCollectionId is not None:
                # Set ISPyB to started
                self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
                    EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                               processingPrograms=self.processingPrograms, isAnom=False,
                                                               timeStart=self.timeStart)
        self.edPluginExecXDSAPPAnom.synchronize()
        xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput
        # Run XSCALE even if XSCALE.LP is present
        strPathXscaleLpAnom = self.runXscale(self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True)
        if self.doAnomAndNonanom:
            self.edPluginExecXDSAPPNoanom.synchronize()
            xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput
            strPathXscaleLpNoanom = self.runXscale(self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True)
        self.timeEnd = time.localtime()
        # Upload to ISPyB
        if self.dataInput.dataCollectionId is not None:
            # Check if we should use XDS_ASCII_to_XML.pl
            if self.useXdsAsciiToXml:
                # Only for anom runs
                self.runXdsAsciiToXml(xsDataResultXDSAPPAnom,
                                      self.dataInput.dataCollectionId.value,
                                      self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
            else:
                self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPAnom, processDirectory, template,
                                   strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd,
                                   self.dataInput.dataCollectionId.value,
                                   self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
                if self.hasUploadedAnomResultsToISPyB:
                    self.screen("Anom results uploaded to ISPyB")
                else:
                    self.ERROR("Could not upload anom results to ISPyB!")



                if self.doAnomAndNonanom:
                    self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPNoanom, processDirectory, template,
                                       strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd,
                                       self.dataInput.dataCollectionId.value,
                                       self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom)
                    if self.hasUploadedNoanomResultsToISPyB:
                        self.screen("Noanom results uploaded to ISPyB")
                    else:
                        self.ERROR("Could not upload noanom results to ISPyB!")
 def storeResultsInISPyB(self, _strSubject, _strMessage):
     strSubject = _strSubject
     strMessage = _strMessage
     xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput().getResultCharacterisation()
     if self.bIsEigerDetector:
         xsDataResultCharacterisation = self.makeNumberOfImagesMultipleOf100(xsDataResultCharacterisation)
     self.xsDataResultMXCuBE.setCharacterisationResult(xsDataResultCharacterisation)
     xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput().getResultControlISPyB()
     if xsDataResultControlISPyB != None:
         self.xsDataResultMXCuBE.setScreeningId(xsDataResultControlISPyB.getScreeningId())
     if xsDataResultCharacterisation != None:
         self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation
         strPathCharacterisationResult = os.path.join(self.getWorkingDirectory(), "CharacterisationResult.xml")
         xsDataResultCharacterisation.exportToFile(strPathCharacterisationResult)
         self.xsDataResultMXCuBE.setListOfOutputFiles(XSDataString(strPathCharacterisationResult))
         # For the moment, create "DNA" style output directory
         strPathToDNAFileDirectory = self.createDNAFileDirectoryPath(xsDataResultCharacterisation)
         xsDataDictionaryLogFile = None
         if self.createDNAFileDirectory(strPathToDNAFileDirectory):
             xsDataDictionaryLogFile = self.createOutputFileDictionary(
                 xsDataResultCharacterisation, strPathToDNAFileDirectory
             )
         strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strPathToDNAFileDirectory)
         if self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory):
             xsDataDictionaryLogFile = self.createOutputFileDictionary(
                 xsDataResultCharacterisation, strPyArchPathToDNAFileDirectory
             )
         self.xsDataResultMXCuBE.setOutputFileDictionary(xsDataDictionaryLogFile)
         if xsDataResultCharacterisation.getStatusMessage():
             strMessage += "\n\n"
             strMessage += xsDataResultCharacterisation.getStatusMessage().getValue()
         if xsDataResultCharacterisation.getShortSummary():
             strMessage += "\n\n"
             strMessage += xsDataResultCharacterisation.getShortSummary().getValue()
         self.sendEmail(strSubject, strMessage)
         # Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run
         bRunExecOutputHTML = False
         xsDataInputMXCuBE = self.getDataInput()
         xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan()
         if xsDataDiffractionPlan is not None and xsDataDiffractionPlan.getStrategyOption() is not None:
             strStrategyOption = xsDataDiffractionPlan.getStrategyOption().getValue()
             if strStrategyOption.find("-DamPar") != -1:
                 bRunExecOutputHTML = False
         if (self.edPluginExecOutputHTML is not None) and bRunExecOutputHTML:
             self.edPluginExecOutputHTML.setDataInput(
                 XSDataFile(XSDataString(strPathToDNAFileDirectory)), "dnaFileDirectory"
             )
             self.edPluginExecOutputHTML.execute()
         # Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory
         xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult()
         if xsDataIntegrationResult:
             listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult()
             for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult:
                 if xsDataIntegrationSubWedgeResult.getBestfilePar() is not None:
                     strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar().getValue()
                     # Put the file one directory above the mxCuBE v1.3 plugin working directory:
                     strDir = os.path.dirname(self.getWorkingDirectory())
                     strPath = os.path.join(strDir, "bestfile.par")
                     EDUtilsFile.writeFile(strPath, strBestfilePar)
                     break
         # Execute plugin which creates a simple HTML page
         self.executeSimpleHTML(xsDataResultCharacterisation)
         # Upload the best wilson plot path to ISPyB
         strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(xsDataResultCharacterisation)
         strBestWilsonPlotPyarchPath = None
         if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None:
             # Copy wilson path to Pyarch
             strBestWilsonPlotPyarchPath = os.path.join(
                 strPyArchPathToDNAFileDirectory, os.path.basename(strBestWilsonPlotPath)
             )
             if not os.path.exists(strBestWilsonPlotPyarchPath):
                 if not os.path.exists(os.path.dirname(strBestWilsonPlotPyarchPath)):
                     os.makedirs(os.path.dirname(strBestWilsonPlotPyarchPath), 755)
                 shutil.copy(strBestWilsonPlotPath, strBestWilsonPlotPyarchPath)
             self.DEBUG("Best wilson pyarch path: %s " % strBestWilsonPlotPyarchPath)
             if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None:
                 xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath()
                 xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = (
                     self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId
                 )
                 xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString(strBestWilsonPlotPyarchPath)
                 edPluginSetBestWilsonPlotPath = self.loadPlugin(
                     "EDPluginISPyBSetBestWilsonPlotPathv1_4", "ISPyBSetBestWilsonPlotPath"
                 )
                 edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath
                 edPluginSetBestWilsonPlotPath.executeSynchronous()
         # Only for the ESRF:
         if EDUtilsPath.isESRF():
             # For EXI: create workflow entry with one workflow step
             self.edPluginStoreWorkflow = self.loadPlugin(self.strPluginStoreWorkflow)
             self.edPluginStoreWorkflowStep = self.loadPlugin(self.strPluginStoreWorkflowStep)
             self.edPluginUpdateDataCollectionGroupWorkflowId = self.loadPlugin(
                 self.strUpdateDataCollectionGroupWorkflowId
             )
             xsDataISPyBWorkflow = XSDataISPyBWorkflow()
             xsDataISPyBWorkflow.workflowType = XSDataString("Characterisation")
             xsDataISPyBWorkflow.workflowTitle = XSDataString("Characterisation")
             xsDataInputISPyBStoreWorkflow = XSDataInputISPyBStoreWorkflow()
             xsDataInputISPyBStoreWorkflow.workflow = xsDataISPyBWorkflow
             self.edPluginStoreWorkflow.dataInput = xsDataInputISPyBStoreWorkflow
             self.edPluginStoreWorkflow.executeSynchronous()
             if (
                 self.edPluginStoreWorkflow.dataOutput is not None
                 and self.edPluginStoreWorkflow.dataOutput.workflowId is not None
             ):
                 workflowId = self.edPluginStoreWorkflow.dataOutput.workflowId.value
                 # Update data collection group
                 xsDataInputISPyBUpdateDataCollectionGroupWorkflowId = (
                     XSDataInputISPyBUpdateDataCollectionGroupWorkflowId()
                 )
                 xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.workflowId = XSDataInteger(workflowId)
                 xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.fileLocation = XSDataString(
                     os.path.dirname(self.xsDataFirstImage.path.value)
                 )
                 xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.fileName = XSDataString(
                     os.path.basename(self.xsDataFirstImage.path.value)
                 )
                 self.edPluginUpdateDataCollectionGroupWorkflowId.dataInput = (
                     xsDataInputISPyBUpdateDataCollectionGroupWorkflowId
                 )
                 self.edPluginUpdateDataCollectionGroupWorkflowId.executeSynchronous()
                 xsDataInputISPyBStoreWorkflowStep = XSDataInputISPyBStoreWorkflowStep()
                 xsDataInputISPyBStoreWorkflowStep.workflowId = XSDataInteger(workflowId)
                 xsDataInputISPyBStoreWorkflowStep.workflowStepType = XSDataString("Characterisation")
                 xsDataInputISPyBStoreWorkflowStep.status = XSDataString("Success")
                 if strBestWilsonPlotPyarchPath is not None:
                     xsDataInputISPyBStoreWorkflowStep.imageResultFilePath = XSDataString(
                         strBestWilsonPlotPyarchPath
                     )
                 xsDataInputISPyBStoreWorkflowStep.htmlResultFilePath = XSDataString(strPyArchPathToDNAFileDirectory)
                 if self.edPluginExecSimpleHTML.dataOutput is not None:
                     strResultFilePath = self.edPluginExecSimpleHTML.dataOutput.pathToJsonFile.path.value
                     # strPyarchResultFilePath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strResultFilePath)
                     xsDataInputISPyBStoreWorkflowStep.resultFilePath = XSDataString(strResultFilePath)
                 self.edPluginStoreWorkflowStep.dataInput = xsDataInputISPyBStoreWorkflowStep
                 self.edPluginStoreWorkflowStep.executeSynchronous()
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG('EDPluginControlAutoPROCv1_0.process starting')

        directory = None
        template = None
        imageNoStart = None
        imageNoEnd = None
        pathToStartImage = None
        pathToEndImage = None
        userName = os.environ["USER"]
        beamline = "unknown"
        proposal = "unknown"

        # If we have a data collection id, use it
        if self.dataInput.dataCollectionId is not None:
            # Recover the data collection from ISPyB
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            identifier = str(self.dataInput.dataCollectionId.value)
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            self.edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
            directory = ispybDataCollection.imageDirectory
            if EDUtilsPath.isEMBL():
                template = ispybDataCollection.fileTemplate.replace("%05d", "#" * 5)
            elif EDUtilsPath.isMAXIV():
                template = ispybDataCollection.fileTemplate
            else:
                template = ispybDataCollection.fileTemplate.replace("%04d", "####")
            if self.dataInput.fromN is None:
                imageNoStart = ispybDataCollection.startImageNumber
            else:
                imageNoStart = self.dataInput.fromN.value
            if self.dataInput.toN is None:
                imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
            else:
                imageNoEnd = self.dataInput.toN.value
#            # DEBUG we set the end image to 20 in order to speed up things
#            self.warning("End image set to 20 (was {0})".format(imageNoEnd))
#            imageNoEnd = 20
            pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
        else:
            identifier = str(int(time.time()))
            directory = self.dataInput.dirN.path.value
            template = self.dataInput.templateN.value
            imageNoStart = self.dataInput.fromN.value
            imageNoEnd = self.dataInput.toN.value
            if EDUtilsPath.isEMBL():
                fileTemplate = template.replace("#####", "%05d")
            else:
                fileTemplate = template.replace("####", "%04d")
            pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
            pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)

        # Try to get proposal from path
        if EDUtilsPath.isESRF():
            listDirectory = directory.split(os.sep)
            try:
                if listDirectory[1] == "data":
                    if listDirectory[2] == "visitor":
                        beamline = listDirectory[4]
                        proposal = listDirectory[3]
                    else:
                        beamline = listDirectory[2]
                        proposal = listDirectory[4]
            except:
                beamline = "unknown"
                proposal = userName


        if imageNoEnd - imageNoStart < 8:
            error_message = "There are fewer than 8 images, aborting"
            self.addErrorMessage(error_message)
            self.ERROR(error_message)
            self.setFailure()
            return

        # Process directory
        if self.dataInput.processDirectory is not None:
            processDirectory = self.dataInput.processDirectory.path.value
        else:
            processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")

        # Make results directory
        if EDUtilsPath.isALBA():
            _processDirectory = "_".join(pathToStartImage.split('_')[:-1])
            from datetime import datetime
            _id = datetime.now().strftime('%Y%m%d_%H%M%S')
            self.resultsDirectory = os.path.join(_processDirectory, "autoPROC_%s" % _id)
        else:
            self.resultsDirectory = os.path.join(processDirectory, "results")
            if not os.path.exists(self.resultsDirectory):
                os.makedirs(self.resultsDirectory, 0o755)


        # Create path to pyarch
        if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
                "autoPROC", self.dataInput.dataCollectionId.value)
        else:
            self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
        if self.pyarchDirectory is not None:
            self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
            if not os.path.exists(self.pyarchDirectory):
                try:
                    os.makedirs(self.pyarchDirectory, 0o755)
                except:
                    self.pyarchDirectory = None

        # The resultsDirectory is not used at ALBA (only pyarchDirectory)
        if EDUtilsPath.isALBA():
            self.resultsDirectory = None

        # Determine pyarch prefix
        if EDUtilsPath.isALBA():
            listPrefix = template.split("_")
            self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]),
                                                       listPrefix[-2])
        else:
            listPrefix = template.split("_")
            self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])

        isH5 = False
        if any(beamline in pathToStartImage for beamline in ["id30b"]):
            minSizeFirst = 6000000
            minSizeLast = 6000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
            minSizeFirst = 2000000
            minSizeLast = 2000000
        elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(directory,
                                            self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(directory,
                                          self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True
        else:
            minSizeFirst = 1000000
            minSizeLast = 1000000

        if EDUtilsPath.isMAXIV():
            minSizeFirst = 100000
            minSizeLast = 100000
            pathToStartImage = os.path.join(directory,
                                            self.eiger_template_to_image(template, imageNoStart))
            pathToEndImage = os.path.join(directory,
                                          self.eiger_template_to_image(template, imageNoEnd))
            isH5 = True

        if EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV():
            fWaitFileTimeout = 60  # s
        else:
            fWaitFileTimeout = 3600  # s

        xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
        xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
        self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
        self.edPluginWaitFileFirst.executeSynchronous()
        if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
            strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
            self.addWarningMessage(strWarningMessage)
            self.WARNING(strWarningMessage)

        xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
        xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
        xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
        self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
        self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
        self.edPluginWaitFileLast.executeSynchronous()
        if self.edPluginWaitFileLast.dataOutput.timedOut.value:
            strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
            self.addErrorMessage(strErrorMessage)
            self.ERROR(strErrorMessage)
            self.setFailure()

        self.timeStart = time.localtime()
        if self.dataInput.dataCollectionId is not None:
            # Set ISPyB to running
            if self.doAnom:
                self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingProgram,
                                                             isAnom=True,
                                                             timeStart=self.timeStart)
                self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingProgramStaraniso,
                                                             isAnom=True,
                                                             timeStart=self.timeStart)
            if self.doNoanom:
                self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingProgram,
                                                             isAnom=False,
                                                             timeStart=self.timeStart)
                self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \
                  EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
                                                             processingCommandLine=self.processingCommandLine,
                                                             processingPrograms=self.processingProgramStaraniso,
                                                             isAnom=False,
                                                             timeStart=self.timeStart)


        # Prepare input to execution plugin
        if self.doAnom:
            xsDataInputAutoPROCAnom = XSDataInputAutoPROC()
            xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True)
            xsDataInputAutoPROCAnom.symm = self.dataInput.symm
            xsDataInputAutoPROCAnom.cell = self.dataInput.cell
            xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit
            xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit
        if self.doNoanom:
            xsDataInputAutoPROCNoanom = XSDataInputAutoPROC()
            xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(False)
            xsDataInputAutoPROCNoanom.symm = self.dataInput.symm
            xsDataInputAutoPROCNoanom.cell = self.dataInput.cell
            xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit
            xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit
        xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier()
        xsDataAutoPROCIdentifier.idN = XSDataString(identifier)
        xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory))
        xsDataAutoPROCIdentifier.templateN = XSDataString(template)
        xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart)
        xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd)
        if self.doAnom:
            xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier)
        if self.doNoanom:
            xsDataInputAutoPROCNoanom.addIdentifier(xsDataAutoPROCIdentifier.copy())
        if isH5:
            masterFilePath = os.path.join(directory,
                                          self.eiger_template_to_master(template))
            if self.doAnom:
                xsDataInputAutoPROCAnom.masterH5 = XSDataFile(XSDataString(masterFilePath))
            if self.doNoanom:
                xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(XSDataString(masterFilePath))
        timeStart = time.localtime()
        if self.doAnom:
            self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom
            self.edPluginExecAutoPROCAnom.execute()
        if self.doNoanom:
            self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom
            self.edPluginExecAutoPROCNoanom.execute()
        if self.doAnom:
            self.edPluginExecAutoPROCAnom.synchronize()
        if self.doNoanom:
            self.edPluginExecAutoPROCNoanom.synchronize()
        timeEnd = time.localtime()

        # Upload to ISPyB
        if self.doAnom:
            self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False, proposal, timeStart, timeEnd)
            self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True, proposal, timeStart, timeEnd)
        if self.doNoanom:
            self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, False, proposal, timeStart, timeEnd)
            self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, True, proposal, timeStart, timeEnd)