def createInputStoreAutoProc(dataCollectionId,
                              integrationId,
                              isAnomalous=False,
                              programId=None,
                              status="SUCCESS",
                              timeStart=None,
                              timeEnd=None,
                              processingCommandLine=None,
                              processingPrograms=None):
     autoProcContainer = AutoProcContainer()
     integrationContainer = AutoProcIntegrationContainer()
     scalingContainer = AutoProcScalingContainer()
     integration = AutoProcIntegration()
     if integrationId is not None:
         integration.autoProcIntegrationId = integrationId
     integration.anomalous = isAnomalous
     integrationContainer.AutoProcIntegration = integration
     image = Image()
     image.dataCollectionId = dataCollectionId
     integrationContainer.Image = image
     scalingContainer.AutoProcIntegrationContainer = integrationContainer
     programContainer = AutoProcProgramContainer()
     programContainer.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
         programId=programId,
         status=status,
         timeStart=timeStart,
         timeEnd=timeEnd,
         processingCommandLine=processingCommandLine,
         processingPrograms=processingPrograms)
     autoProcContainer.AutoProcProgramContainer = programContainer
     autoProcContainer.AutoProcScalingContainer = scalingContainer
     inputStoreAutoProc = XSDataInputStoreAutoProc()
     inputStoreAutoProc.AutoProcContainer = autoProcContainer
     return inputStoreAutoProc
 def createInputStoreAutoProc(dataCollectionId, integrationId, isAnomalous=False,
                              programId=None, status="SUCCESS", timeStart=None, timeEnd=None,
                              processingCommandLine=None, processingPrograms=None):
     autoProcContainer = AutoProcContainer()
     integrationContainer = AutoProcIntegrationContainer()
     scalingContainer = AutoProcScalingContainer()
     integration = AutoProcIntegration()
     if integrationId is not None:
         integration.autoProcIntegrationId = integrationId
     integration.anomalous = isAnomalous
     integrationContainer.AutoProcIntegration = integration
     image = Image()
     image.dataCollectionId = dataCollectionId
     integrationContainer.Image = image
     scalingContainer.AutoProcIntegrationContainer = integrationContainer
     programContainer = AutoProcProgramContainer()
     programContainer.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
         programId=programId, status=status, timeStart=timeStart, timeEnd=timeEnd,
         processingCommandLine=processingCommandLine, processingPrograms=processingPrograms)
     autoProcContainer.AutoProcProgramContainer = programContainer
     autoProcContainer.AutoProcScalingContainer = scalingContainer
     inputStoreAutoProc = XSDataInputStoreAutoProc()
     inputStoreAutoProc.AutoProcContainer = autoProcContainer
     return inputStoreAutoProc
    def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, timeStart, timeEnd):
        if isAnom:
            anomString = "anom"
        else:
            anomString = "noanom"

        # Copy dataFiles to results directory
        for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
            trunc, suffix = os.path.splitext(dataFile.path.value)
            newFileName = trunc + "_" + anomString + suffix
            shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName))

        # Read the generated ISPyB xml file - if any
        if edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
            autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)

            # "Fix" certain entries in the ISPyB xml file
            autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
            for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
                if isAnom:
                    autoProcScalingStatistics.anomalous = True
                else:
                    autoProcScalingStatistics.anomalous = False
            autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
            autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
            if isAnom:
                autoProcIntegration.anomalous = True
            else:
                autoProcIntegration.anomalous = False
            image = autoProcIntegrationContainer.Image
            image.dataCollectionId = self.dataInput.dataCollectionId.value
            autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
            autoProcProgram = autoProcProgramContainer.AutoProcProgram
            autoProcProgram.processingPrograms = "XIA2_DIALS"
            autoProcProgram.processingStatus = True
            autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
            autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
            autoProcProgramContainer.AutoProcProgramAttachment = []
            # Upload the log file to ISPyB
            if edPluginExecXia2DIALS.dataOutput.logFile is not None:
                pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
                pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
                shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchFileName
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Log"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Upload the summary file to ISPyB
            if edPluginExecXia2DIALS.dataOutput.summary is not None:
                pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
                pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
                shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchFileName
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Log"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Create a pdf file of the html page
            if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
                pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
                pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.pdf"
                # Convert the xia2.html to xia2.pdf
                xsDataInputHTML2PDF = XSDataInputHTML2PDF()
                xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(pathToHtmlFile)))
                xsDataInputHTML2PDF.paperSize = XSDataString("A4")
                xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
                edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
                edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
                edPluginHTML2Pdf.executeSynchronous()
                pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
                shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchFileName))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchFileName
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Log"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Copy all log files
            for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
                pathToLogFile = logFile.path.value
                if pathToLogFile.endswith(".log"):
                    pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile)
                    shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
                    autoProcProgramAttachment = AutoProcProgramAttachment()
                    autoProcProgramAttachment.fileName = pyarchFileName
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                    autoProcProgramAttachment.fileType = "Log"
                    autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Copy data files
            for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
                pathToDataFile = dataFile.path.value
                if pathToDataFile.endswith(".mtz"):
                    pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile)
                    shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName))
                    autoProcProgramAttachment = AutoProcProgramAttachment()
                    autoProcProgramAttachment.fileName = pyarchFileName
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                    autoProcProgramAttachment.fileType = "Result"
                    autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Upload the xml to ISPyB
            xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
            xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
            edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
            edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
            edPluginStoreAutoproc.executeSynchronous()
Esempio n. 4
0
 for iIndex, strArg in enumerate(sys.argv[1:]):
     strarg = strArg.lower()
     if strarg == "--verbose":
         EDVerbose.setVerboseOn()
     elif strarg == "--debug":
         EDVerbose.setVerboseDebugOn()
     strArgPath = strArg
     if os.path.dirname(strArgPath) == "":
         strArgPath = os.path.join(strCwd, strArg)
     strArgPathAbsolute = os.path.abspath(strArgPath)
     if os.path.exists(strArgPathAbsolute):
         listPaths.append(strArgPathAbsolute)
 if listPaths == []:
     EDVerbose.ERROR("No valid XML file given as input!")
     EDVerbose.ERROR("Arguments: %r" % sys.argv)
     EDVerbose.ERROR("Usage: edna-autoproc2ispyb path_to_xml_file [--debug]")
     sys.exit(1)
 for strPath in listPaths:
     EDVerbose.screen("Starting EDNA AutoProc -> ISPyB for file %s" % strPath)
     xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
     strXMLAutoProcContainer = EDUtilsFile.readFile(strPath)
     xsDataAutoProcContainer = AutoProcContainer.parseString(strXMLAutoProcContainer)
     if xsDataAutoProcContainer is None:
         EDVerbose.WARNING("Couldn't parse file %s" % strPath)
     else:
         xsDataInputStoreAutoProc.setAutoProcContainer(xsDataAutoProcContainer)
         edPluginISPyBStoreAutoProc = EDFactoryPluginStatic.loadPlugin("EDPluginISPyBStoreAutoProcv1_4")
         edPluginISPyBStoreAutoProc.setDataInput(xsDataInputStoreAutoProc)
         edPluginISPyBStoreAutoProc.connectSUCCESS(successAction)
         edPluginISPyBStoreAutoProc.connectFAILURE(failureAction)
         edPluginISPyBStoreAutoProc.executeSynchronous()
Esempio n. 5
0
     strArgPath = strArg
     if os.path.dirname(strArgPath) == "":
         strArgPath = os.path.join(strCwd, strArg)
     strArgPathAbsolute = os.path.abspath(strArgPath)
     if os.path.exists(strArgPathAbsolute):
         listPaths.append(strArgPathAbsolute)
 if listPaths == []:
     EDVerbose.ERROR("No valid XML file given as input!")
     EDVerbose.ERROR("Arguments: %r" % sys.argv)
     EDVerbose.ERROR(
         "Usage: edna-autoproc2ispyb path_to_xml_file [--debug]")
     sys.exit(1)
 for strPath in listPaths:
     EDVerbose.screen("Starting EDNA AutoProc -> ISPyB for file %s" %
                      strPath)
     xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
     strXMLAutoProcContainer = EDUtilsFile.readFile(strPath)
     xsDataAutoProcContainer = AutoProcContainer.parseString(
         strXMLAutoProcContainer)
     if xsDataAutoProcContainer is None:
         EDVerbose.WARNING("Couldn't parse file %s" % strPath)
     else:
         if xsDataAutoProcContainer.AutoProcProgramContainer.AutoProcProgram.processingStatus == "True":
             xsDataAutoProcContainer.AutoProcProgramContainer.AutoProcProgram.processingStatus = "SUCCESS"
         xsDataInputStoreAutoProc.setAutoProcContainer(
             xsDataAutoProcContainer)
         edPluginISPyBStoreAutoProc = EDFactoryPluginStatic.loadPlugin(
             "EDPluginISPyBStoreAutoProcv1_4")
         edPluginISPyBStoreAutoProc.setDataInput(xsDataInputStoreAutoProc)
         edPluginISPyBStoreAutoProc.executeSynchronous()
         if edPluginISPyBStoreAutoProc.isFailure():
Esempio n. 6
0
    def postProcess(self, _edObject=None):
        EDPluginControl.postProcess(self)
        self.DEBUG("EDPluginControlCrystFELv1_0.postProcess")
        """
        autoProcScalingStatisticsId : integer optional
        scalingStatisticsType : string
        comments : string
        resolutionLimitLow : float
        resolutionLimitHigh : float
        rMerge : float
        rMeasWithinIPlusIMinus : float
        rMeasAllIPlusIMinus : float
        rPimWithinIPlusIMinus : float
        rPimAllIPlusIMinus : float
        fractionalPartialBias : float
        nTotalObservations : integer
        ntotalUniqueObservations : integer
        meanIOverSigI : float
        completeness : float
        multiplicity : float
        anomalousCompleteness : float
        anomalousMultiplicity : float
        recordTimeStamp : string
        anomalous : boolean
        autoProcScalingId : integer
        ccHalf : float
        ccAno : float
        sigAno : float
        isa : float
        completenessSpherical : float
        anomalousCompletenessSpherical : float
        completenessEllipsoidal : float
        anomalousCompletenessEllipsoidal : float
        """
        """
        overallCompl : XSDataDouble
        overallRed : XSDataDouble
        overallSnr : XSDataDouble
        overallRsplit : XSDataDouble
        overallCC
        """

        output = AutoProcContainer()
        autoproc = AutoProc()
        autoproc.spaceGroup = str(self.dataInput.spaceGroup.value)
        output.AutoProc = autoproc

        scaling_container = AutoProcScalingContainer()
        scaling = AutoProcScaling()
        scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S",
                                                time.localtime())
        scaling_container.AutoProcScaling = scaling

        #Open the cell file and get the cell parameters

        cell_file = open(self.dataInput.cellFile.value, "r")
        cell_params = {}
        for line in cell_file.readlines():
            if line.startswith("a ="):
                cell_params["a"] = float(line[4:9])
            elif line.startswith("b ="):
                cell_params["b"] = float(line[4:9])
            elif line.startswith("c ="):
                cell_params["c"] = float(line[4:9])
            elif line.startswith("al ="):
                cell_params["alpha"] = float(line[4:9])
            elif line.startswith("be ="):
                cell_params["beta"] = float(line[4:9])
            elif line.startswith("ga ="):
                cell_params["gamma"] = float(line[4:9])
        cell_file.close()

        autoproc.refinedCell_a = cell_params["a"]
        autoproc.refinedCell_b = cell_params["b"]
        autoproc.refinedCell_c = cell_params["c"]
        autoproc.refinedCell_alpha = cell_params["alpha"]
        autoproc.refinedCell_beta = cell_params["beta"]
        autoproc.refinedCell_gamma = cell_params["gamma"]

        overall_stats = AutoProcScalingStatistics()
        overall_stats.scalingStatisticsType = "innerShell"
        overall_stats.ccHalf = float(
            self.post_process_plugin.dataOutput.overallCC.value)
        overall_stats.resolutionLimitLow = float(
            self.post_process_plugin.dataOutput.resolutionLimitLow.value)
        overall_stats.resolutionLimitHigh = float(
            self.post_process_plugin.dataOutput.resolutionLimitHigh.value)
        overall_stats.completeness = float(
            self.post_process_plugin.dataOutput.overallCompl.value)
        #TODO get rMerge: otherwise results are not displayed in EXI
        overall_stats.rMerge = 0

        scaling_container.AutoProcScalingStatistics.append(overall_stats)

        integration_container = AutoProcIntegrationContainer()
        image = Image()
        image.dataCollectionId = self.dataInput.dataCollectionId.value
        integration_container.Image = image

        integration = AutoProcIntegration()
        if self.integration_id is not None:
            integration.autoProcIntegrationId = self.integration_id
            integration.cell_a = cell_params["a"]
            integration.cell_b = cell_params["b"]
            integration.cell_c = cell_params["c"]
            integration.cell_alpha = cell_params["alpha"]
            integration.cell_beta = cell_params["beta"]
            integration.cell_gamma = cell_params["gamma"]
            integration.anomalous = 0

        # done with the integration
        integration_container.AutoProcIntegration = integration
        scaling_container.AutoProcIntegrationContainer = integration_container

        program_container = AutoProcProgramContainer()
        program_container.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
            programId=self.program_id,
            status="SUCCESS",
            timeStart=self.timeStart,
            timeEnd=self.timeEnd,
            processingCommandLine=self.processingCommandLine,
            processingPrograms=self.processingPrograms)

        output.AutoProcProgramContainer = program_container
        output.AutoProcScalingContainer = scaling_container

        for log_file in self.post_process_plugin.dataOutput.logFiles:
            pathToLogFile = log_file.path.value
            pyarchFileName = os.path.join(self.pyarchDirectory,
                                          os.path.basename(pathToLogFile))
            #shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))

            autoproc_program_attachment = AutoProcProgramAttachment()
            autoproc_program_attachment.fileName = os.path.basename(
                pyarchFileName)
            autoproc_program_attachment.filePath = os.path.dirname(
                pyarchFileName)
            autoproc_program_attachment.fileType = "Log"
            program_container.addAutoProcProgramAttachment(
                autoproc_program_attachment)
            self.screen("Result file %s uploaded to ISPyB" %
                        os.path.basename(pyarchFileName))

        for data_file in self.post_process_plugin.dataOutput.dataFiles:
            pathToDataFile = data_file.path.value
            pyarchFileName = os.path.join(self.pyarchDirectory,
                                          os.path.basename(pathToDataFile))
            #shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))

            autoproc_program_attachment = AutoProcProgramAttachment()
            autoproc_program_attachment.fileName = os.path.basename(
                pyarchFileName)
            autoproc_program_attachment.filePath = os.path.dirname(
                pyarchFileName)
            autoproc_program_attachment.fileType = "Result"
            program_container.addAutoProcProgramAttachment(
                autoproc_program_attachment)
            self.screen("Result file %s uploaded to ISPyB" %
                        os.path.basename(pyarchFileName))

        ispyb_input = XSDataInputStoreAutoProc()
        ispyb_input.AutoProcContainer = output

        self.store_autoproc_plugin.dataInput = ispyb_input
        self.store_autoproc_plugin.executeSynchronous()

        if self.store_autoproc_plugin.isFailure():
            self.ERROR("Could not upload results to ispyb!")
        else:
            self.hasUploadedAnomResultsToISPyB = True
            self.screen("Results uploaded to ISPyB")

        xsDataInput = XSDataInputISPyBUpdateDataCollectionGroupComment()
        xsDataInput.newComment = XSDataString(
            self.post_process_plugin.dataOutput.comment.value)
        xsDataInput.dataCollectionId = self.dataInput.dataCollectionId
        self.store_dc_comment_plugin.dataInput = xsDataInput
        self.executePluginSynchronous(self.store_dc_comment_plugin)
Esempio n. 7
0
    def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template,
                                       strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId,
                                       integrationId=None, programId=None):

        # Parse log file
        dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value)
        dictXscale = self.parseXscaleLp(strPathXscaleLp)

        xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
        autoProcContainer = AutoProcContainer()


        # AutoProc
        autoProc = AutoProc()
        autoProc.spaceGroup = dictLog["spaceGroup"]
        autoProc.refinedCell_a = dictLog["cellA"]
        autoProc.refinedCell_b = dictLog["cellB"]
        autoProc.refinedCell_c = dictLog["cellC"]
        autoProc.refinedCell_alpha = dictLog["cellAlpha"]
        autoProc.refinedCell_beta = dictLog["cellBeta"]
        autoProc.refinedCell_gamma = dictLog["cellGamma"]
        autoProcContainer.AutoProc = autoProc

        # AutoProcIntegrationContainer
        autoProcIntegrationContainer = AutoProcIntegrationContainer()
        autoProcIntegration = AutoProcIntegration()
        autoProcIntegration.autoProcIntegrationId = integrationId
        autoProcIntegration.cell_a = dictLog["cellA"]
        autoProcIntegration.cell_b = dictLog["cellB"]
        autoProcIntegration.cell_c = dictLog["cellC"]
        autoProcIntegration.cell_alpha = dictLog["cellAlpha"]
        autoProcIntegration.cell_beta = dictLog["cellBeta"]
        autoProcIntegration.cell_gamma = dictLog["cellGamma"]
        autoProcIntegration.anomalous = isAnom

        image = Image()
        image.dataCollectionId = dataCollectionId
        autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration
        autoProcIntegrationContainer.Image = image


        # Scaling container
        if xsDataResultXDSAPP.correctLP is not None:
            isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value)
        else:
            isa = None
        autoProcScalingContainer = AutoProcScalingContainer()
        autoProcScaling = AutoProcScaling()
        autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        autoProcScalingContainer.AutoProcScaling = autoProcScaling
        for scalingStatisticsType in dictXscale:
            autoProcScalingStatistics = AutoProcScalingStatistics()
            autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType
            autoProcScalingStatistics.anomalous = isAnom
            for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]:
                setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute])
            if scalingStatisticsType == "overall" and isa is not None:
                autoProcScalingStatistics.isa = isa
            autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics)
        autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer
        autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer

        # Program
        autoProcProgramContainer = AutoProcProgramContainer()

        autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
                programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd,
                processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
        autoProcProgramContainer.AutoProcProgram = autoProcProgram

        # XDSAPP log and result files
        if xsDataResultXDSAPP.logFile is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value,
                               "xdsapp", "log", isAnom, attachmentType="Log")
        if xsDataResultXDSAPP.pointlessLog is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value,
                               "pointless", "log", isAnom, attachmentType="Log")
        if xsDataResultXDSAPP.phenixXtriageLog is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value,
                               "xtriage", "log", isAnom, attachmentType="Log")
        if xsDataResultXDSAPP.correctLP is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value,
                               "CORRECT", "LP", isAnom, attachmentType="Log")
        if xsDataResultXDSAPP.XDS_ASCII_HKL is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value,
                               "XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True)
        if xsDataResultXDSAPP.XDS_INP is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value,
                               "XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True)
        for mtz_F in xsDataResultXDSAPP.mtz_F:
            basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0]
            self.addAttachment(autoProcProgramContainer, mtz_F.path.value,
                               basenameMtz_F, "mtz", isAnom, attachmentType="Result")
        for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus:
            basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0]
            self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value,
                               basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result")
#        for mtz_I in xsDataResultXDSAPP.mtz_I:
#            basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0]
#            self.addAttachment(autoProcProgramContainer, mtz_I.path.value,
#                               basenameMtz_I, "mtz", isAnom, attachmentType="Result")
#        for hkl in xsDataResultXDSAPP.hkl:
#            basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0]
#            self.addAttachment(autoProcProgramContainer, hkl.path.value,
#                               basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True)
#        for cv in xsDataResultXDSAPP.cv:
#            basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0]
#            self.addAttachment(autoProcProgramContainer, cv.path.value,
#                               basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True)

        if os.path.exists(strPathXscaleLp):
            self.addAttachment(autoProcProgramContainer, strPathXscaleLp,
                               "XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result")
        autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer
        xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
        return xsDataInputStoreAutoProc
    def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId,
                      integrationId):
        successUpload = False

        if isAnom:
            anomString = "anom"
        else:
            anomString = "noanom"

        # Read the generated ISPyB xml file - if any
        if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
            autoProcContainer = AutoProcContainer.parseFile(
                edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)

            # "Fix" certain entries in the ISPyB xml file
            autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
            for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
                if isAnom:
                    autoProcScalingStatistics.anomalous = True
                else:
                    autoProcScalingStatistics.anomalous = False
                # Convert from fraction to %
                autoProcScalingStatistics.rMerge *= 100.0
            autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
            autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
            autoProcIntegration.autoProcIntegrationId = integrationId
            if isAnom:
                autoProcIntegration.anomalous = True
            else:
                autoProcIntegration.anomalous = False
            image = autoProcIntegrationContainer.Image
            image.dataCollectionId = self.dataInput.dataCollectionId.value
            autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
            autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
                programId=programId,
                status="SUCCESS",
                timeStart=self.timeStart,
                timeEnd=self.timeEnd,
                processingCommandLine=self.processingCommandLine,
                processingPrograms=self.processingPrograms)
            autoProcProgramContainer.AutoProcProgram = autoProcProgram
            autoProcProgramContainer.AutoProcProgramAttachment = []
            # Upload the log file to ISPyB
            if edPluginExecXia2DIALS.dataOutput.logFile is not None:
                pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
                pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
                shutil.copy(pathToLogFile,
                            os.path.join(self.pyarchDirectory, pyarchFileName))
                shutil.copy(
                    pathToLogFile,
                    os.path.join(self.resultsDirectory, pyarchFileName))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchFileName
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Log"
                autoProcProgramContainer.addAutoProcProgramAttachment(
                    autoProcProgramAttachment)
            # Upload the summary file to ISPyB
            if edPluginExecXia2DIALS.dataOutput.summary is not None:
                pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
                pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
                shutil.copy(pathToSummaryFile,
                            os.path.join(self.pyarchDirectory, pyarchFileName))
                shutil.copy(
                    pathToSummaryFile,
                    os.path.join(self.resultsDirectory, pyarchFileName))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchFileName
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Log"
                autoProcProgramContainer.addAutoProcProgramAttachment(
                    autoProcProgramAttachment)
            # Create a pdf file of the html page
            if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
                pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
                pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html"
                shutil.copy(pathToHtmlFile,
                            os.path.join(self.pyarchDirectory, pyarchFileName))
                shutil.copy(
                    pathToHtmlFile,
                    os.path.join(self.resultsDirectory, pyarchFileName))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchFileName
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Log"
                autoProcProgramContainer.addAutoProcProgramAttachment(
                    autoProcProgramAttachment)
            # Copy all log files
            for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
                pathToLogFile = logFile.path.value
                pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(
                    pathToLogFile)
                # Copy all log files to results:
                shutil.copy(
                    pathToLogFile,
                    os.path.join(self.resultsDirectory, pyarchFileName))
                # Only copy .log files to pyarch
                if pathToLogFile.endswith(".log"):
                    shutil.copy(
                        pathToLogFile,
                        os.path.join(self.pyarchDirectory, pyarchFileName))
                    autoProcProgramAttachment = AutoProcProgramAttachment()
                    autoProcProgramAttachment.fileName = pyarchFileName
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                    autoProcProgramAttachment.fileType = "Log"
                    autoProcProgramContainer.addAutoProcProgramAttachment(
                        autoProcProgramAttachment)
            # Copy data files
            for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
                pathToDataFile = dataFile.path.value
                if pathToDataFile.endswith(".mtz"):
                    pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(
                        pathToDataFile)
                    shutil.copy(
                        pathToDataFile,
                        os.path.join(self.pyarchDirectory, pyarchFileName))
                    shutil.copy(
                        pathToDataFile,
                        os.path.join(self.resultsDirectory, pyarchFileName))
                    autoProcProgramAttachment = AutoProcProgramAttachment()
                    autoProcProgramAttachment.fileName = pyarchFileName
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                    autoProcProgramAttachment.fileType = "Result"
                    autoProcProgramContainer.addAutoProcProgramAttachment(
                        autoProcProgramAttachment)
            # Upload the xml to ISPyB
            xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
            xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
            edPluginStoreAutoproc = self.loadPlugin(
                "EDPluginISPyBStoreAutoProcv1_4",
                "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
            edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
            edPluginStoreAutoproc.executeSynchronous()
            successUpload = not edPluginStoreAutoproc.isFailure()
        else:
            # Copy dataFiles to results directory
            for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
                trunc, suffix = os.path.splitext(dataFile.path.value)
                newFileName = os.path.basename(
                    trunc) + "_" + anomString + suffix
                shutil.copy(dataFile.path.value,
                            os.path.join(self.resultsDirectory, newFileName))
        return successUpload
    def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template,
                                       strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId,
                                       integrationId=None, programId=None):

        # Parse log file
        dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value)
        dictXscale = self.parseXscaleLp(strPathXscaleLp)

        # Create path to pyarch
        self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
        self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
        if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
            os.makedirs(self.pyarchDirectory, 0o755)

        # Determine pyarch prefix
        listPrefix = template.split("_")
        self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])


        xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
        autoProcContainer = AutoProcContainer()


        # AutoProc
        autoProc = AutoProc()
        autoProc.spaceGroup = dictLog["spaceGroup"]
        autoProc.refinedCell_a = dictLog["cellA"]
        autoProc.refinedCell_b = dictLog["cellB"]
        autoProc.refinedCell_c = dictLog["cellC"]
        autoProc.refinedCell_alpha = dictLog["cellAlpha"]
        autoProc.refinedCell_beta = dictLog["cellBeta"]
        autoProc.refinedCell_gamma = dictLog["cellGamma"]
        autoProcContainer.AutoProc = autoProc

        # AutoProcIntegrationContainer
        autoProcIntegrationContainer = AutoProcIntegrationContainer()
        autoProcIntegration = AutoProcIntegration()
        autoProcIntegration.autoProcIntegrationId = integrationId
        autoProcIntegration.cell_a = dictLog["cellA"]
        autoProcIntegration.cell_b = dictLog["cellB"]
        autoProcIntegration.cell_c = dictLog["cellC"]
        autoProcIntegration.cell_alpha = dictLog["cellAlpha"]
        autoProcIntegration.cell_beta = dictLog["cellBeta"]
        autoProcIntegration.cell_gamma = dictLog["cellGamma"]
        autoProcIntegration.anomalous = isAnom

        image = Image()
        image.dataCollectionId = dataCollectionId
        autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration
        autoProcIntegrationContainer.Image = image


        # Scaling container
        if xsDataResultXDSAPP.correctLP is not None:
            isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value)
        else:
            isa = None
        autoProcScalingContainer = AutoProcScalingContainer()
        autoProcScaling = AutoProcScaling()
        autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        autoProcScalingContainer.AutoProcScaling = autoProcScaling
        for scalingStatisticsType in dictXscale:
            autoProcScalingStatistics = AutoProcScalingStatistics()
            autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType
            autoProcScalingStatistics.anomalous = isAnom
            for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]:
                setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute])
            if scalingStatisticsType == "overall" and isa is not None:
                autoProcScalingStatistics.isa = isa
            autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics)
        autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer
        autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer

        # Program
        autoProcProgramContainer = AutoProcProgramContainer()

        autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
                programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd,
                processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
        autoProcProgramContainer.AutoProcProgram = autoProcProgram

        # Attached files
        pyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(processDirectory)
        pyarchResultPath = os.path.join(pyarchPath, "results")
        if not os.path.exists(pyarchResultPath):
            os.makedirs(pyarchResultPath, 0o755)

        # XDSAPP log and result files
        if xsDataResultXDSAPP.logFile is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value,
                               "xdsapp", "log", isAnom, attachmentType="Log")
        if xsDataResultXDSAPP.pointlessLog is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value,
                               "pointless", "log", isAnom, attachmentType="Log")
        if xsDataResultXDSAPP.phenixXtriageLog is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value,
                               "xtriage", "log", isAnom, attachmentType="Log")
        if xsDataResultXDSAPP.correctLP is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value,
                               "CORRECT", "LP", isAnom, attachmentType="Log")
        if xsDataResultXDSAPP.XDS_ASCII_HKL is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value,
                               "XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True)
        if xsDataResultXDSAPP.XDS_INP is not None:
            self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value,
                               "XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True)
        for mtz_F in xsDataResultXDSAPP.mtz_F:
            basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0]
            self.addAttachment(autoProcProgramContainer, mtz_F.path.value,
                               basenameMtz_F, "mtz", isAnom, attachmentType="Result")
        for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus:
            basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0]
            self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value,
                               basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result")
#        for mtz_I in xsDataResultXDSAPP.mtz_I:
#            basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0]
#            self.addAttachment(autoProcProgramContainer, mtz_I.path.value,
#                               basenameMtz_I, "mtz", isAnom, attachmentType="Result")
#        for hkl in xsDataResultXDSAPP.hkl:
#            basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0]
#            self.addAttachment(autoProcProgramContainer, hkl.path.value,
#                               basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True)
#        for cv in xsDataResultXDSAPP.cv:
#            basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0]
#            self.addAttachment(autoProcProgramContainer, cv.path.value,
#                               basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True)

        if os.path.exists(strPathXscaleLp):
            self.addAttachment(autoProcProgramContainer, strPathXscaleLp,
                               "XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result")
        autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer
        xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
        return xsDataInputStoreAutoProc
    def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId, integrationId):
        successUpload = False

        if isAnom:
            anomString = "anom"
        else:
            anomString = "noanom"

        # Read the generated ISPyB xml file - if any
        if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None:
            autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value)

            # "Fix" certain entries in the ISPyB xml file
            autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
            for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
                if isAnom:
                    autoProcScalingStatistics.anomalous = True
                else:
                    autoProcScalingStatistics.anomalous = False
                # Convert from fraction to %
                autoProcScalingStatistics.rMerge *= 100.0
            autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
            autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
            autoProcIntegration.autoProcIntegrationId = integrationId
            if isAnom:
                autoProcIntegration.anomalous = True
            else:
                autoProcIntegration.anomalous = False
            image = autoProcIntegrationContainer.Image
            image.dataCollectionId = self.dataInput.dataCollectionId.value
            autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
            autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
                    programId=programId, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd,
                    processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
            autoProcProgramContainer.AutoProcProgram = autoProcProgram
            autoProcProgramContainer.AutoProcProgramAttachment = []
            # Upload the log file to ISPyB
            if edPluginExecXia2DIALS.dataOutput.logFile is not None:
                pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value
                pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log"
                shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
                shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchFileName
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Log"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Upload the summary file to ISPyB
            if edPluginExecXia2DIALS.dataOutput.summary is not None:
                pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value
                pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log"
                shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName))
                shutil.copy(pathToSummaryFile, os.path.join(self.resultsDirectory, pyarchFileName))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchFileName
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Log"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Create a pdf file of the html page
            if edPluginExecXia2DIALS.dataOutput.htmlFile is not None:
                pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value
                pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html"
                shutil.copy(pathToHtmlFile, os.path.join(self.pyarchDirectory, pyarchFileName))
                shutil.copy(pathToHtmlFile, os.path.join(self.resultsDirectory, pyarchFileName))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchFileName
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Log"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Copy all log files
            for logFile in edPluginExecXia2DIALS.dataOutput.logFiles:
                pathToLogFile = logFile.path.value
                pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile)
                # Copy all log files to results:
                shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName))
                # Only copy .log files to pyarch
                if pathToLogFile.endswith(".log"):
                    shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName))
                    autoProcProgramAttachment = AutoProcProgramAttachment()
                    autoProcProgramAttachment.fileName = pyarchFileName
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                    autoProcProgramAttachment.fileType = "Log"
                    autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Copy data files
            for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
                pathToDataFile = dataFile.path.value
                if pathToDataFile.endswith(".mtz"):
                    pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile)
                    shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName))
                    shutil.copy(pathToDataFile, os.path.join(self.resultsDirectory, pyarchFileName))
                    autoProcProgramAttachment = AutoProcProgramAttachment()
                    autoProcProgramAttachment.fileName = pyarchFileName
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                    autoProcProgramAttachment.fileType = "Result"
                    autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Upload the xml to ISPyB
            xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
            xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
            edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString))
            edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc
            edPluginStoreAutoproc.executeSynchronous()
            successUpload = not edPluginStoreAutoproc.isFailure()
        else:
            # Copy dataFiles to results directory
            for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles:
                trunc, suffix = os.path.splitext(dataFile.path.value)
                newFileName = os.path.basename(trunc) + "_" + anomString + suffix
                shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName))
        return successUpload
    def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, proposal, timeStart, timeEnd):
        if isAnom:
            anomString = "anom"
        else:
            anomString = "noanom"
        # Read the generated ISPyB xml file - if any
        if edPluginExecAutoPROC.dataOutput.ispybXML is not None:
            autoProcContainer = AutoProcContainer.parseFile(edPluginExecAutoPROC.dataOutput.ispybXML.path.value)

            # "Fix" certain entries in the ISPyB xml file
            autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
            for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
                if isAnom:
                    autoProcScalingStatistics.anomalous = True
                else:
                    autoProcScalingStatistics.anomalous = False
                # Convert from fraction to %
                autoProcScalingStatistics.rMerge *= 100.0
            autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
            image = autoProcIntegrationContainer.Image
            image.dataCollectionId = self.dataInput.dataCollectionId.value
            autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
            if isAnom:
                autoProcIntegration.anomalous = True
            else:
                autoProcIntegration.anomalous = False
            autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
            autoProcProgram = autoProcProgramContainer.AutoProcProgram
            autoProcProgram.processingPrograms = "autoPROC"
            autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
            autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
            for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment:
                if autoProcProgramAttachment.fileName == "summary.html":
                    summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
                    # Replace opidXX with user name
                    htmlSummary = open(summaryHtmlPath).read()
                    userString1 = "User      : {0} (".format(os.environ["USER"])
                    userString2 = "User      : {0} (".format(proposal)
                    htmlSummary = htmlSummary.replace(userString1, userString2)
                    open(summaryHtmlPath, "w").write(htmlSummary)
                    # Convert the summary.html to summary.pdf
                    xsDataInputHTML2PDF = XSDataInputHTML2PDF()
                    xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath)))
                    xsDataInputHTML2PDF.paperSize = XSDataString("A3")
                    xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
                    edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
                    edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
                    edPluginHTML2Pdf.executeSynchronous()
                    pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
                    pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile)
                    # Copy file to results directory and pyarch
                    shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile))
                    shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile))
                    autoProcProgramAttachment.fileName = pyarchPdfFile
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                elif autoProcProgramAttachment.fileName == "truncate-unique.mtz":
                    pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
                    pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString)
                    shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
                    shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
                    autoProcProgramAttachment.fileName = pyarchFile
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                else:
                    pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
                    pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName
                    shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
                    shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
                    autoProcProgramAttachment.fileName = pyarchFile
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
            # Add XSCALE.LP file if present
            processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value
            pathToXSCALELog = os.path.join(processDirectory, "xscale_XSCALE.LP")
            if os.path.exists(pathToXSCALELog):
                pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString)
                shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog))
                shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchXSCALELog
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Result"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Add XDS_ASCII.HKL if present and gzip it
            pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL")
            if os.path.exists(pathToXdsAsciiHkl):
                pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString)
                f_in = open(pathToXdsAsciiHkl)
                f_out = gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), "wb")
                f_out.writelines(f_in)
                f_out.close()
                f_in.close()
                shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Result"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Add log file
            pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value
            autoPROClog = open(pathToLogFile).read()
            userString1 = "User      : {0} (".format(os.environ["USER"])
            userString2 = "User      : {0} (".format(proposal)
            autoPROClog = autoPROClog.replace(userString1, userString2)
            open(pathToLogFile, "w").write(autoPROClog)
            pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString)
            shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile))
            shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile))
            autoProcProgramAttachment = AutoProcProgramAttachment()
            autoProcProgramAttachment.fileName = pyarchLogFile
            autoProcProgramAttachment.filePath = self.pyarchDirectory
            autoProcProgramAttachment.fileType = "Log"
            autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)

            # Upload the xml to ISPyB
            xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
            xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
            if isAnom:
                self.edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc
                self.edPluginStoreAutoprocAnom.executeSynchronous()
            else:
                self.edPluginStoreAutoprocNoanom.dataInput = xsDataInputStoreAutoProc
                self.edPluginStoreAutoprocNoanom.executeSynchronous()
    def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, isStaraniso, proposal, timeStart, timeEnd):
        if isAnom:
            anomString = "anom"
        else:
            anomString = "noanom"
        if isStaraniso:
            staranisoString = "_staraniso"
        else:
            staranisoString = ""
        # Read the generated ISPyB xml file
        pathToISPyBXML = None
        if isStaraniso:
            if edPluginExecAutoPROC.dataOutput.ispybXML_staraniso is not None:
                pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML_staraniso.path.value
        elif edPluginExecAutoPROC.dataOutput.ispybXML is not None:
            pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML.path.value
        if pathToISPyBXML is not None:
            autoProcContainer = AutoProcContainer.parseFile(pathToISPyBXML)
            # "Fix" certain entries in the ISPyB xml file
            autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
            for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
                if isAnom:
                    autoProcScalingStatistics.anomalous = True
                else:
                    autoProcScalingStatistics.anomalous = False
                # Convert from fraction to %
                autoProcScalingStatistics.rMerge *= 100.0
                autoProcScalingStatistics.rMeasWithinIPlusIMinus *= 100
                autoProcScalingStatistics.rMeasAllIPlusIMinus *= 100
                autoProcScalingStatistics.rPimWithinIPlusIMinus *= 100
                autoProcScalingStatistics.rPimAllIPlusIMinus *= 100
            autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
            image = autoProcIntegrationContainer.Image
            if self.dataInput.dataCollectionId is not None:
                image.dataCollectionId = self.dataInput.dataCollectionId.value
            autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
            autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
            autoProcProgram = autoProcProgramContainer.AutoProcProgram
            if isAnom:
                autoProcIntegration.anomalous = True
                if isStaraniso:
                    autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnomStaraniso
                    autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnomStaraniso
                else:
                    autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnom
                    autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnom
            else:
                autoProcIntegration.anomalous = False
                if isStaraniso:
                    autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanomStaraniso
                    autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanomStaraniso
                else:
                    autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanom
                    autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanom
            if self.reprocess:
                autoProcProgram.processingPrograms = "autoPROC" + staranisoString + " reprocess"
            else:
                autoProcProgram.processingPrograms = "autoPROC" + staranisoString
            autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
            autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
            autoProcProgram.processingStatus = "SUCCESS"
            # EDNA-245 - remove "truncate_{early,late}-unique.mtz" from autoProcProgramContainer.AutoProcProgramAttachment
            autoProcProgramContainer.AutoProcProgramAttachment[:] = [x for x in autoProcProgramContainer.AutoProcProgramAttachment if not self.matchesTruncateEarlyLate(x.fileName) ]
            for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment:
                if autoProcProgramAttachment.fileName == "summary.html":
                    # Check if summary_inlined.html exists
                    summaryInlinedHtmlPath = os.path.join(autoProcProgramAttachment.filePath, "summary_inlined.html")
                    if os.path.exists(summaryInlinedHtmlPath):
                        summaryName = "summary_inlined"
                        summaryHtmlPath = summaryInlinedHtmlPath
                    else:
                        summaryName = "summary"
                        summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
                    # Replace opidXX with user name
                    htmlSummary = open(summaryHtmlPath).read()
                    userString1 = "User      : {0} (".format(os.environ["USER"])
                    userString2 = "User      : {0} (".format(proposal)
                    htmlSummary = htmlSummary.replace(userString1, userString2)
                    open(summaryHtmlPath, "w").write(htmlSummary)
                    # Upload summary.html
                    pathtoFile = summaryHtmlPath
                    pyarchFile = self.pyarchPrefix + "_{0}_{1}.html".format(anomString, summaryName)
                    if not pyarchFile in self.listPyarchFile:
                        if self.resultsDirectory:
                            shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
                            self.listPyarchFile.append(pyarchFile)
                    if self.pyarchDirectory is not None:
                        shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
                        autoProcProgramAttachment.fileName = os.path.basename(pyarchFile)
                        autoProcProgramAttachment.filePath = self.pyarchDirectory
                        autoProcProgramAttachment.fileType = "Log"

                    if summaryName == "summary":
                        # Convert the summary.html to summary.pdf
                        xsDataInputHTML2PDF = XSDataInputHTML2PDF()
                        xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath)))
                        xsDataInputHTML2PDF.paperSize = XSDataString("A3")
                        xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
                        edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
                        edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
                        edPluginHTML2Pdf.executeSynchronous()
                        pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
                        pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile)
                        # Copy file to results directory and pyarch
                        if self.resultsDirectory:
                            shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile))
                        if self.pyarchDirectory is not None:
                            shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile))
                            autoProcProgramAttachmentPdf = AutoProcProgramAttachment()
                            autoProcProgramAttachmentPdf.fileName = pyarchPdfFile
                            autoProcProgramAttachmentPdf.filePath = self.pyarchDirectory
                            autoProcProgramAttachmentPdf.fileType = "Log"
                            autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachmentPdf)
                elif autoProcProgramAttachment.fileName == "truncate-unique.mtz":
                    pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
                    pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString)
                    if self.resultsDirectory:
                        shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
                    if self.pyarchDirectory is not None:
                        shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
                        autoProcProgramAttachment.fileName = pyarchFile
                        autoProcProgramAttachment.filePath = self.pyarchDirectory
                else:
                    pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
                    pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName
                    if self.resultsDirectory:
                        shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
                    if self.pyarchDirectory is not None:
                        shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
                        autoProcProgramAttachment.fileName = pyarchFile
                        autoProcProgramAttachment.filePath = self.pyarchDirectory
            # Add XSCALE.LP file if present
            processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value
            pathToXSCALELog = os.path.join(processDirectory, "XSCALE.LP")
            if os.path.exists(pathToXSCALELog):
                pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString)
                if self.resultsDirectory:
                    shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog))
                if self.pyarchDirectory is not None:
                    shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog))
                    autoProcProgramAttachment = AutoProcProgramAttachment()
                    autoProcProgramAttachment.fileName = pyarchXSCALELog
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                    autoProcProgramAttachment.fileType = "Result"
                    autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Add XDS_ASCII.HKL if present and gzip it
            pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL")
            if os.path.exists(pathToXdsAsciiHkl) and self.pyarchDirectory is not None:
                pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString)
                with open(pathToXdsAsciiHkl, 'rb') as f_in:
                    with gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), 'wb') as f_out:
                        shutil.copyfileobj(f_in, f_out)
                if self.resultsDirectory:
                    shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Result"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Add log file
            pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value
            autoPROClog = open(pathToLogFile).read()
            userString1 = "User      : {0} (".format(os.environ["USER"])
            userString2 = "User      : {0} (".format(proposal)
            autoPROClog = autoPROClog.replace(userString1, userString2)
            open(pathToLogFile, "w").write(autoPROClog)
            pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString)
            if self.resultsDirectory:
                shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile))
            if self.pyarchDirectory is not None:
                shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchLogFile
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Log"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Add report.pdf
            pathToRepordPdf = None
            if isStaraniso and edPluginExecAutoPROC.dataOutput.reportPdf_staraniso is not None:
                pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf_staraniso.path.value
            elif edPluginExecAutoPROC.dataOutput.reportPdf is not None:
                pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf.path.value
            if pathToRepordPdf is not None:
                pyarchReportFile = self.pyarchPrefix + "_{0}_{1}".format(anomString, os.path.basename(pathToRepordPdf))
                if self.resultsDirectory:
                    shutil.copy(pathToRepordPdf, os.path.join(self.resultsDirectory, pyarchReportFile))
                if self.pyarchDirectory is not None:
                    shutil.copy(pathToRepordPdf, os.path.join(self.pyarchDirectory, pyarchReportFile))
                    autoProcProgramAttachment = AutoProcProgramAttachment()
                    autoProcProgramAttachment.fileName = pyarchReportFile
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                    autoProcProgramAttachment.fileType = "Log"
                    autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)

            # Upload the xml to ISPyB
            xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
            xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
            edPluginStoreAutoprocAnom = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "ISPyBStoreAutoProcv1_4_{0}{1}".format(anomString, staranisoString))
            edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc
            edPluginStoreAutoprocAnom.executeSynchronous()
            isSuccess = not edPluginStoreAutoprocAnom.isFailure()
            if isSuccess:
                self.screen("{0}{1} results uploaded to ISPyB".format(anomString, staranisoString))
                if isAnom:
                    if isStaraniso:
                        self.hasUploadedAnomStaranisoResultsToISPyB = True
                    else:
                        self.hasUploadedAnomResultsToISPyB = True
                else:
                    if isStaraniso:
                        self.hasUploadedNoanomStaranisoResultsToISPyB = True
                    else:
                        self.hasUploadedNoanomResultsToISPyB = True
            else:
                self.screen("Could not upload {0}{1} results to ISPyB".format(anomString, staranisoString))
Esempio n. 13
0
    def postProcess(self, _edObject = None):
        EDPluginControl.postProcess(self)
        self.DEBUG("EDPluginControlAutoproc.postProcess")

        #Now that we have executed the whole thing we need to create
        #the suitable ISPyB plugin input and serialize it to the file
        #we've been given as input
        output = AutoProcContainer()

        # AutoProc attr
        autoproc = AutoProc()

        xdsout = self.xds_first.dataOutput
        if xdsout.sg_number is not None: # and it should not
            autoproc.spaceGroup = SPACE_GROUP_NAMES[xdsout.sg_number.value]
        autoproc.refinedCell_a = xdsout.cell_a.value
        autoproc.refinedCell_b = xdsout.cell_b.value
        autoproc.refinedCell_c = xdsout.cell_c.value
        autoproc.refinedCell_alpha = xdsout.cell_alpha.value
        autoproc.refinedCell_beta = xdsout.cell_beta.value
        autoproc.refinedCell_gamma = xdsout.cell_gamma.value

        output.AutoProc = autoproc

        # scaling container and all the things that go in
        scaling_container_noanom = AutoProcScalingContainer()
        scaling = AutoProcScaling()
        scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

        scaling_container_noanom.AutoProcScaling = scaling

        # NOANOM PATH
        xscale_stats_noanom = self.xscale_generate.dataOutput.stats_noanom_merged
        inner_stats_noanom = xscale_stats_noanom.completeness_entries[0]
        outer_stats_noanom = xscale_stats_noanom.completeness_entries[-1]

        # use the previous shell's res as low res, if available
        prev_res = self.low_resolution_limit
        try:
            prev_res = xscale_stats_noanom.completeness_entries[-2].outer_res.value
        except IndexError:
            pass
        total_stats_noanom = xscale_stats_noanom.total_completeness

        stats = _create_scaling_stats(inner_stats_noanom, 'innerShell',
                                      self.low_resolution_limit, False)
        overall_low = stats.resolutionLimitLow
        scaling_container_noanom.AutoProcScalingStatistics.append(stats)

        stats = _create_scaling_stats(outer_stats_noanom, 'outerShell',
                                      prev_res, False)
        overall_high = stats.resolutionLimitHigh
        scaling_container_noanom.AutoProcScalingStatistics.append(stats)
        stats = _create_scaling_stats(total_stats_noanom, 'overall',
                                      self.low_resolution_limit, False)
        stats.resolutionLimitLow = overall_low
        stats.resolutionLimitHigh = overall_high
        scaling_container_noanom.AutoProcScalingStatistics.append(stats)

        integration_container_noanom = AutoProcIntegrationContainer()
        image = Image()
        image.dataCollectionId = self.dataInput.data_collection_id.value
        integration_container_noanom.Image = image

        integration_noanom = AutoProcIntegration()
        if self.integration_id_noanom is not None:
            integration_noanom.autoProcIntegrationId = self.integration_id_noanom
        crystal_stats =  self.parse_xds_noanom.dataOutput
        integration_noanom.cell_a = crystal_stats.cell_a.value
        integration_noanom.cell_b = crystal_stats.cell_b.value
        integration_noanom.cell_c = crystal_stats.cell_c.value
        integration_noanom.cell_alpha = crystal_stats.cell_alpha.value
        integration_noanom.cell_beta = crystal_stats.cell_beta.value
        integration_noanom.cell_gamma = crystal_stats.cell_gamma.value
        integration_noanom.anomalous = 0

        # done with the integration
        integration_container_noanom.AutoProcIntegration = integration_noanom
        scaling_container_noanom.AutoProcIntegrationContainer = integration_container_noanom

        # ANOM PATH
        scaling_container_anom = AutoProcScalingContainer()
        scaling = AutoProcScaling()
        scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

        scaling_container_anom.AutoProcScaling = scaling

        xscale_stats_anom = self.xscale_generate.dataOutput.stats_anom_merged
        inner_stats_anom = xscale_stats_anom.completeness_entries[0]
        outer_stats_anom = xscale_stats_anom.completeness_entries[-1]

        # use the previous shell's res as low res if available
        prev_res = self.low_resolution_limit
        try:
            prev_res = xscale_stats_anom.completeness_entries[-2].outer_res.value
        except IndexError:
            pass
        total_stats_anom = xscale_stats_anom.total_completeness

        stats = _create_scaling_stats(inner_stats_anom, 'innerShell',
                                      self.low_resolution_limit, True)
        overall_low = stats.resolutionLimitLow
        scaling_container_anom.AutoProcScalingStatistics.append(stats)

        stats = _create_scaling_stats(outer_stats_anom, 'outerShell',
                                      prev_res, True)
        overall_high = stats.resolutionLimitHigh
        scaling_container_anom.AutoProcScalingStatistics.append(stats)
        stats = _create_scaling_stats(total_stats_anom, 'overall',
                                      self.low_resolution_limit, True)
        stats.resolutionLimitLow = overall_low
        stats.resolutionLimitHigh = overall_high
        scaling_container_anom.AutoProcScalingStatistics.append(stats)


        integration_container_anom = AutoProcIntegrationContainer()
        image = Image()
        image.dataCollectionId = self.dataInput.data_collection_id.value
        integration_container_anom.Image = image

        integration_anom = AutoProcIntegration()
        crystal_stats =  self.parse_xds_anom.dataOutput
        if self.integration_id_anom is not None:
            integration_anom.autoProcIntegrationId = self.integration_id_anom
        integration_anom.cell_a = crystal_stats.cell_a.value
        integration_anom.cell_b = crystal_stats.cell_b.value
        integration_anom.cell_c = crystal_stats.cell_c.value
        integration_anom.cell_alpha = crystal_stats.cell_alpha.value
        integration_anom.cell_beta = crystal_stats.cell_beta.value
        integration_anom.cell_gamma = crystal_stats.cell_gamma.value
        integration_anom.anomalous = 1

        # done with the integration
        integration_container_anom.AutoProcIntegration = integration_anom
        scaling_container_anom.AutoProcIntegrationContainer = integration_container_anom


        # ------ NO ANOM / ANOM end



        program_container = AutoProcProgramContainer()
        program_container.AutoProcProgram = AutoProcProgram()
        program_container.AutoProcProgram.processingCommandLine = ' '.join(sys.argv)
        program_container.AutoProcProgram.processingPrograms = 'edna-fastproc'

        # now for the generated files. There's some magic to do with
        # their paths to determine where to put them on pyarch
        pyarch_path = None
        # Note: the path is in the form /data/whatever

        # remove the edna-autoproc-import suffix
        original_files_dir = self.file_conversion.dataInput.output_directory.value
        #files_dir, _ = os.path.split(original_files_dir)
        files_dir = original_files_dir

        # the whole transformation is fragile!
        if files_dir.startswith('/data/visitor'):
            # We might get empty elements at the head/tail of the list
            tokens = [elem for elem in files_dir.split(os.path.sep)
                      if len(elem) > 0]
            pyarch_path = os.path.join('/data/pyarch',
                                       tokens[3], tokens[2],
                                       *tokens[4:])
        else:
            # We might get empty elements at the head/tail of the list
            tokens = [elem for elem in files_dir.split(os.path.sep)
                      if len(elem) > 0]
            if tokens[2] == 'inhouse':
                pyarch_path = os.path.join('/data/pyarch', tokens[1],
                                           *tokens[3:])
        if pyarch_path is not None:
            pyarch_path = pyarch_path.replace('PROCESSED_DATA', 'RAW_DATA')
            try:
                os.makedirs(pyarch_path)
            except OSError:
                # dir already exists, may happen when testing
                EDVerbose.screen('Target directory on pyarch ({0}) already exists, ignoring'.format(pyarch_path))

            file_list = []
            # we can now copy the files to this dir
            for f in os.listdir(original_files_dir):
                current = os.path.join(original_files_dir, f)
                if not os.path.isfile(current):
                    continue
                if not os.path.splitext(current)[1].lower() in ISPYB_UPLOAD_EXTENSIONS:
                    continue
                new_path = os.path.join(pyarch_path, f)
                file_list.append(new_path)
                shutil.copyfile(current,
                                new_path)
            # now add those to the ispyb upload
            for path in file_list:
                dirname, filename = os.path.split(path)
                attach = AutoProcProgramAttachment()
                attach.fileType = "Result"
                attach.fileName = filename
                attach.filePath = dirname
                program_container.AutoProcProgramAttachment.append(attach)


        program_container.AutoProcProgram.processingStatus = True
        output.AutoProcProgramContainer = program_container

        # first with anom

        output.AutoProcScalingContainer = scaling_container_anom

        ispyb_input = XSDataInputStoreAutoProc()
        ispyb_input.AutoProcContainer = output


        with open(self.dataInput.output_file.path.value, 'w') as f:
            f.write(ispyb_input.marshal())

        # store results in ispyb
        self.store_autoproc_anom.dataInput = ispyb_input
        t0=time.time()
        self.store_autoproc_anom.executeSynchronous()
        self.stats['ispyb_upload'] = time.time() - t0

        with open(self.log_file_path, 'w') as f:
            json.dump(self.stats, f)

        if self.store_autoproc_anom.isFailure():
            self.ERROR('could not send results to ispyb')
        else:
            # store the autoproc ID as a filename in the
            # fastproc_integration_ids directory
            os.mknod(os.path.join(self.autoproc_ids_dir, str(self.integration_id_anom)), 0755)
        # then noanom stats

        output.AutoProcScalingContainer = scaling_container_noanom

        ispyb_input = XSDataInputStoreAutoProc()
        ispyb_input.AutoProcContainer = output


        with open(self.dataInput.output_file.path.value, 'w') as f:
            f.write(ispyb_input.marshal())

        # store results in ispyb
        self.store_autoproc_noanom.dataInput = ispyb_input
        t0=time.time()
        self.store_autoproc_noanom.executeSynchronous()
        self.stats['ispyb_upload'] = time.time() - t0

        with open(self.log_file_path, 'w') as f:
            json.dump(self.stats, f)

        if self.store_autoproc_noanom.isFailure():
            self.ERROR('could not send results to ispyb')
        else:
            # store the autoproc id
            os.mknod(os.path.join(self.autoproc_ids_dir, str(self.integration_id_noanom)), 0755)
Esempio n. 14
0
    def postProcess(self, _edObject=None):
        EDPluginControl.postProcess(self)
        self.DEBUG("EDPluginControlAutoproc.postProcess")

        #Now that we have executed the whole thing we need to create
        #the suitable ISPyB plugin input and serialize it to the file
        #we've been given as input
        output = AutoProcContainer()

        # AutoProc attr
        autoproc = AutoProc()

        xdsout = self.xds_first.dataOutput
        if xdsout.sg_number is not None:  # and it should not
            autoproc.spaceGroup = SPACE_GROUP_NAMES[xdsout.sg_number.value]
        autoproc.refinedCell_a = xdsout.cell_a.value
        autoproc.refinedCell_b = xdsout.cell_b.value
        autoproc.refinedCell_c = xdsout.cell_c.value
        autoproc.refinedCell_alpha = xdsout.cell_alpha.value
        autoproc.refinedCell_beta = xdsout.cell_beta.value
        autoproc.refinedCell_gamma = xdsout.cell_gamma.value

        output.AutoProc = autoproc

        # scaling container and all the things that go in
        scaling_container_noanom = AutoProcScalingContainer()
        scaling = AutoProcScaling()
        scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S",
                                                time.localtime())

        scaling_container_noanom.AutoProcScaling = scaling

        # NOANOM PATH
        xscale_stats_noanom = self.xscale_generate.dataOutput.stats_noanom_merged
        inner_stats_noanom = xscale_stats_noanom.completeness_entries[0]
        outer_stats_noanom = xscale_stats_noanom.completeness_entries[-1]

        # use the previous shell's res as low res, if available
        prev_res = self.low_resolution_limit
        try:
            prev_res = xscale_stats_noanom.completeness_entries[
                -2].outer_res.value
        except IndexError:
            pass
        total_stats_noanom = xscale_stats_noanom.total_completeness

        stats = _create_scaling_stats(inner_stats_noanom, 'innerShell',
                                      self.low_resolution_limit, False)
        overall_low = stats.resolutionLimitLow
        scaling_container_noanom.AutoProcScalingStatistics.append(stats)

        stats = _create_scaling_stats(outer_stats_noanom, 'outerShell',
                                      prev_res, False)
        overall_high = stats.resolutionLimitHigh
        scaling_container_noanom.AutoProcScalingStatistics.append(stats)
        stats = _create_scaling_stats(total_stats_noanom, 'overall',
                                      self.low_resolution_limit, False)
        stats.resolutionLimitLow = overall_low
        stats.resolutionLimitHigh = overall_high
        scaling_container_noanom.AutoProcScalingStatistics.append(stats)

        integration_container_noanom = AutoProcIntegrationContainer()
        image = Image()
        image.dataCollectionId = self.dataInput.data_collection_id.value
        integration_container_noanom.Image = image

        integration_noanom = AutoProcIntegration()
        if self.integration_id_noanom is not None:
            integration_noanom.autoProcIntegrationId = self.integration_id_noanom
        crystal_stats = self.parse_xds_noanom.dataOutput
        integration_noanom.cell_a = crystal_stats.cell_a.value
        integration_noanom.cell_b = crystal_stats.cell_b.value
        integration_noanom.cell_c = crystal_stats.cell_c.value
        integration_noanom.cell_alpha = crystal_stats.cell_alpha.value
        integration_noanom.cell_beta = crystal_stats.cell_beta.value
        integration_noanom.cell_gamma = crystal_stats.cell_gamma.value
        integration_noanom.anomalous = 0

        # done with the integration
        integration_container_noanom.AutoProcIntegration = integration_noanom
        scaling_container_noanom.AutoProcIntegrationContainer = integration_container_noanom

        # ANOM PATH
        scaling_container_anom = AutoProcScalingContainer()
        scaling = AutoProcScaling()
        scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S",
                                                time.localtime())

        scaling_container_anom.AutoProcScaling = scaling

        xscale_stats_anom = self.xscale_generate.dataOutput.stats_anom_merged
        inner_stats_anom = xscale_stats_anom.completeness_entries[0]
        outer_stats_anom = xscale_stats_anom.completeness_entries[-1]

        # use the previous shell's res as low res if available
        prev_res = self.low_resolution_limit
        try:
            prev_res = xscale_stats_anom.completeness_entries[
                -2].outer_res.value
        except IndexError:
            pass
        total_stats_anom = xscale_stats_anom.total_completeness

        stats = _create_scaling_stats(inner_stats_anom, 'innerShell',
                                      self.low_resolution_limit, True)
        overall_low = stats.resolutionLimitLow
        scaling_container_anom.AutoProcScalingStatistics.append(stats)

        stats = _create_scaling_stats(outer_stats_anom, 'outerShell', prev_res,
                                      True)
        overall_high = stats.resolutionLimitHigh
        scaling_container_anom.AutoProcScalingStatistics.append(stats)
        stats = _create_scaling_stats(total_stats_anom, 'overall',
                                      self.low_resolution_limit, True)
        stats.resolutionLimitLow = overall_low
        stats.resolutionLimitHigh = overall_high
        scaling_container_anom.AutoProcScalingStatistics.append(stats)

        integration_container_anom = AutoProcIntegrationContainer()
        image = Image()
        image.dataCollectionId = self.dataInput.data_collection_id.value
        integration_container_anom.Image = image

        integration_anom = AutoProcIntegration()
        crystal_stats = self.parse_xds_anom.dataOutput
        if self.integration_id_anom is not None:
            integration_anom.autoProcIntegrationId = self.integration_id_anom
        integration_anom.cell_a = crystal_stats.cell_a.value
        integration_anom.cell_b = crystal_stats.cell_b.value
        integration_anom.cell_c = crystal_stats.cell_c.value
        integration_anom.cell_alpha = crystal_stats.cell_alpha.value
        integration_anom.cell_beta = crystal_stats.cell_beta.value
        integration_anom.cell_gamma = crystal_stats.cell_gamma.value
        integration_anom.anomalous = 1

        # done with the integration
        integration_container_anom.AutoProcIntegration = integration_anom
        scaling_container_anom.AutoProcIntegrationContainer = integration_container_anom

        # ------ NO ANOM / ANOM end

        program_container = AutoProcProgramContainer()
        program_container.AutoProcProgram = AutoProcProgram()
        program_container.AutoProcProgram.processingCommandLine = ' '.join(
            sys.argv)
        program_container.AutoProcProgram.processingPrograms = 'edna-fastproc'

        # now for the generated files. There's some magic to do with
        # their paths to determine where to put them on pyarch
        pyarch_path = None
        # Note: the path is in the form /data/whatever

        # remove the edna-autoproc-import suffix
        original_files_dir = self.file_conversion.dataInput.output_directory.value
        #files_dir, _ = os.path.split(original_files_dir)
        files_dir = original_files_dir

        # the whole transformation is fragile!
        if files_dir.startswith('/data/visitor'):
            # We might get empty elements at the head/tail of the list
            tokens = [
                elem for elem in files_dir.split(os.path.sep) if len(elem) > 0
            ]
            pyarch_path = os.path.join('/data/pyarch', tokens[3], tokens[2],
                                       *tokens[4:])
        else:
            # We might get empty elements at the head/tail of the list
            tokens = [
                elem for elem in files_dir.split(os.path.sep) if len(elem) > 0
            ]
            if tokens[2] == 'inhouse':
                pyarch_path = os.path.join('/data/pyarch', tokens[1],
                                           *tokens[3:])
        if pyarch_path is not None:
            pyarch_path = pyarch_path.replace('PROCESSED_DATA', 'RAW_DATA')
            try:
                os.makedirs(pyarch_path)
            except OSError:
                # dir already exists, may happen when testing
                EDVerbose.screen(
                    'Target directory on pyarch ({0}) already exists, ignoring'
                    .format(pyarch_path))

            file_list = []
            # we can now copy the files to this dir
            for f in os.listdir(original_files_dir):
                current = os.path.join(original_files_dir, f)
                if not os.path.isfile(current):
                    continue
                if not os.path.splitext(
                        current)[1].lower() in ISPYB_UPLOAD_EXTENSIONS:
                    continue
                new_path = os.path.join(pyarch_path, f)
                file_list.append(new_path)
                shutil.copyfile(current, new_path)
            # now add those to the ispyb upload
            for path in file_list:
                dirname, filename = os.path.split(path)
                attach = AutoProcProgramAttachment()
                attach.fileType = "Result"
                attach.fileName = filename
                attach.filePath = dirname
                program_container.AutoProcProgramAttachment.append(attach)

        program_container.AutoProcProgram.processingStatus = True
        output.AutoProcProgramContainer = program_container

        # first with anom

        output.AutoProcScalingContainer = scaling_container_anom

        ispyb_input = XSDataInputStoreAutoProc()
        ispyb_input.AutoProcContainer = output

        with open(self.dataInput.output_file.path.value, 'w') as f:
            f.write(ispyb_input.marshal())

        # store results in ispyb
        self.store_autoproc_anom.dataInput = ispyb_input
        t0 = time.time()
        self.store_autoproc_anom.executeSynchronous()
        self.stats['ispyb_upload'] = time.time() - t0

        with open(self.log_file_path, 'w') as f:
            json.dump(self.stats, f)

        if self.store_autoproc_anom.isFailure():
            self.ERROR('could not send results to ispyb')
        else:
            # store the autoproc ID as a filename in the
            # fastproc_integration_ids directory
            os.mknod(
                os.path.join(self.autoproc_ids_dir,
                             str(self.integration_id_anom)), 0755)
        # then noanom stats

        output.AutoProcScalingContainer = scaling_container_noanom

        ispyb_input = XSDataInputStoreAutoProc()
        ispyb_input.AutoProcContainer = output

        with open(self.dataInput.output_file.path.value, 'w') as f:
            f.write(ispyb_input.marshal())

        # store results in ispyb
        self.store_autoproc_noanom.dataInput = ispyb_input
        t0 = time.time()
        self.store_autoproc_noanom.executeSynchronous()
        self.stats['ispyb_upload'] = time.time() - t0

        with open(self.log_file_path, 'w') as f:
            json.dump(self.stats, f)

        if self.store_autoproc_noanom.isFailure():
            self.ERROR('could not send results to ispyb')
        else:
            # store the autoproc id
            os.mknod(
                os.path.join(self.autoproc_ids_dir,
                             str(self.integration_id_noanom)), 0755)
    def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, isStaraniso, proposal, timeStart, timeEnd):
        if isAnom:
            anomString = "anom"
        else:
            anomString = "noanom"
        if isStaraniso:
            staranisoString = "_staraniso"
        else:
            staranisoString = ""
        # Read the generated ISPyB xml file
        pathToISPyBXML = None
        if isStaraniso:
            if edPluginExecAutoPROC.dataOutput.ispybXML_staraniso is not None:
                pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML_staraniso.path.value
        elif edPluginExecAutoPROC.dataOutput.ispybXML is not None:
            pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML.path.value
        if pathToISPyBXML is not None:
            autoProcContainer = AutoProcContainer.parseFile(pathToISPyBXML)
            # "Fix" certain entries in the ISPyB xml file
            autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
            for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
                if isAnom:
                    autoProcScalingStatistics.anomalous = True
                else:
                    autoProcScalingStatistics.anomalous = False
                # Convert from fraction to %
                autoProcScalingStatistics.rMerge *= 100.0
            autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
            image = autoProcIntegrationContainer.Image
            if self.dataInput.dataCollectionId is not None:
                image.dataCollectionId = self.dataInput.dataCollectionId.value
            autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
            autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
            autoProcProgram = autoProcProgramContainer.AutoProcProgram
            if isAnom:
                autoProcIntegration.anomalous = True
                if isStaraniso:
                    autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnomStaraniso
                    autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnomStaraniso
                else:
                    autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnom
                    autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnom
            else:
                autoProcIntegration.anomalous = False
                if isStaraniso:
                    autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanomStaraniso
                    autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanomStaraniso
                else:
                    autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanom
                    autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanom
            autoProcProgram.processingPrograms = "autoPROC" + staranisoString
            autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
            autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
            autoProcProgram.processingStatus = "SUCCESS"
            # EDNA-245 - remove "truncate_{early,late}-unique.mtz" from autoProcProgramContainer.AutoProcProgramAttachment
            autoProcProgramContainer.AutoProcProgramAttachment[:] = [x for x in autoProcProgramContainer.AutoProcProgramAttachment if not self.matchesTruncateEarlyLate(x.fileName) ]
            for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment:
                if autoProcProgramAttachment.fileName == "summary.html":
                    # Check if summary_inlined.html exists
                    summaryInlinedHtmlPath = os.path.join(autoProcProgramAttachment.filePath, "summary_inlined.html")
                    if os.path.exists(summaryInlinedHtmlPath):
                        summaryName = "summary_inlined"
                        summaryHtmlPath = summaryInlinedHtmlPath
                    else:
                        summaryName = "summary"
                        summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
                    # Replace opidXX with user name
                    htmlSummary = open(summaryHtmlPath).read()
                    userString1 = "User      : {0} (".format(os.environ["USER"])
                    userString2 = "User      : {0} (".format(proposal)
                    htmlSummary = htmlSummary.replace(userString1, userString2)
                    open(summaryHtmlPath, "w").write(htmlSummary)
                    # Upload summary.html
                    pathtoFile = summaryHtmlPath
                    pyarchFile = self.pyarchPrefix + "_{0}_{1}.html".format(anomString, summaryName)
                    if not pyarchFile in self.listPyarchFile:
                        shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
                        self.listPyarchFile.append(pyarchFile)
                    if self.pyarchDirectory is not None:
                        shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
                        autoProcProgramAttachment.fileName = os.path.basename(pyarchFile)
                        autoProcProgramAttachment.filePath = self.pyarchDirectory
                        autoProcProgramAttachment.fileType = "Log"

                    if summaryName == "summary":
                        # Convert the summary.html to summary.pdf
                        xsDataInputHTML2PDF = XSDataInputHTML2PDF()
                        xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath)))
                        xsDataInputHTML2PDF.paperSize = XSDataString("A3")
                        xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
                        edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
                        edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
                        edPluginHTML2Pdf.executeSynchronous()
                        pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
                        pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile)
                        # Copy file to results directory and pyarch
                        shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile))
                        if self.pyarchDirectory is not None:
                            shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile))
                            autoProcProgramAttachmentPdf = AutoProcProgramAttachment()
                            autoProcProgramAttachmentPdf.fileName = pyarchPdfFile
                            autoProcProgramAttachmentPdf.filePath = self.pyarchDirectory
                            autoProcProgramAttachmentPdf.fileType = "Log"
                            autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachmentPdf)
                elif autoProcProgramAttachment.fileName == "truncate-unique.mtz":
                    pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
                    pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString)
                    shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
                    if self.pyarchDirectory is not None:
                        shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
                        autoProcProgramAttachment.fileName = pyarchFile
                        autoProcProgramAttachment.filePath = self.pyarchDirectory
                else:
                    pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
                    pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName
                    shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
                    if self.pyarchDirectory is not None:
                        shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
                        autoProcProgramAttachment.fileName = pyarchFile
                        autoProcProgramAttachment.filePath = self.pyarchDirectory
            # Add XSCALE.LP file if present
            processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value
            pathToXSCALELog = os.path.join(processDirectory, "XSCALE.LP")
            if os.path.exists(pathToXSCALELog):
                pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString)
                shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog))
                if self.pyarchDirectory is not None:
                    shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog))
                    autoProcProgramAttachment = AutoProcProgramAttachment()
                    autoProcProgramAttachment.fileName = pyarchXSCALELog
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                    autoProcProgramAttachment.fileType = "Result"
                    autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Add XDS_ASCII.HKL if present and gzip it
            pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL")
            if os.path.exists(pathToXdsAsciiHkl) and self.pyarchDirectory is not None:
                pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString)
                f_in = open(pathToXdsAsciiHkl)
                f_out = gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), "wb")
                f_out.writelines(f_in)
                f_out.close()
                f_in.close()
                shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Result"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Add log file
            pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value
            autoPROClog = open(pathToLogFile).read()
            userString1 = "User      : {0} (".format(os.environ["USER"])
            userString2 = "User      : {0} (".format(proposal)
            autoPROClog = autoPROClog.replace(userString1, userString2)
            open(pathToLogFile, "w").write(autoPROClog)
            pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString)
            shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile))
            if self.pyarchDirectory is not None:
                shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile))
                autoProcProgramAttachment = AutoProcProgramAttachment()
                autoProcProgramAttachment.fileName = pyarchLogFile
                autoProcProgramAttachment.filePath = self.pyarchDirectory
                autoProcProgramAttachment.fileType = "Log"
                autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
            # Add report.pdf
            pathToRepordPdf = None
            if isStaraniso and edPluginExecAutoPROC.dataOutput.reportPdf_staraniso is not None:
                pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf_staraniso.path.value
            elif edPluginExecAutoPROC.dataOutput.reportPdf is not None:
                pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf.path.value
            if pathToRepordPdf is not None:
                pyarchReportFile = self.pyarchPrefix + "_{0}_{1}".format(anomString, os.path.basename(pathToRepordPdf))
                shutil.copy(pathToRepordPdf, os.path.join(self.resultsDirectory, pyarchReportFile))
                if self.pyarchDirectory is not None:
                    shutil.copy(pathToRepordPdf, os.path.join(self.pyarchDirectory, pyarchReportFile))
                    autoProcProgramAttachment = AutoProcProgramAttachment()
                    autoProcProgramAttachment.fileName = pyarchReportFile
                    autoProcProgramAttachment.filePath = self.pyarchDirectory
                    autoProcProgramAttachment.fileType = "Log"
                    autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)

            # Upload the xml to ISPyB
            xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
            xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
            edPluginStoreAutoprocAnom = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "ISPyBStoreAutoProcv1_4_{0}{1}".format(anomString, staranisoString))
            edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc
            edPluginStoreAutoprocAnom.executeSynchronous()
            isSuccess = not edPluginStoreAutoprocAnom.isFailure()
            if isSuccess:
                self.screen("{0}{1} results uploaded to ISPyB".format(anomString, staranisoString))
                if isAnom:
                    if isStaraniso:
                        self.hasUploadedAnomStaranisoResultsToISPyB = True
                    else:
                        self.hasUploadedAnomResultsToISPyB = True
                else:
                    if isStaraniso:
                        self.hasUploadedNoanomStaranisoResultsToISPyB = True
                    else:
                        self.hasUploadedNoanomResultsToISPyB = True
            else:
                self.screen("Could not upload {0}{1} results to ISPyB".format(anomString, staranisoString))