def createInputStoreAutoProc(dataCollectionId, integrationId, isAnomalous=False, programId=None, status="SUCCESS", timeStart=None, timeEnd=None, processingCommandLine=None, processingPrograms=None): autoProcContainer = AutoProcContainer() integrationContainer = AutoProcIntegrationContainer() scalingContainer = AutoProcScalingContainer() integration = AutoProcIntegration() if integrationId is not None: integration.autoProcIntegrationId = integrationId integration.anomalous = isAnomalous integrationContainer.AutoProcIntegration = integration image = Image() image.dataCollectionId = dataCollectionId integrationContainer.Image = image scalingContainer.AutoProcIntegrationContainer = integrationContainer programContainer = AutoProcProgramContainer() programContainer.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status=status, timeStart=timeStart, timeEnd=timeEnd, processingCommandLine=processingCommandLine, processingPrograms=processingPrograms) autoProcContainer.AutoProcProgramContainer = programContainer autoProcContainer.AutoProcScalingContainer = scalingContainer inputStoreAutoProc = XSDataInputStoreAutoProc() inputStoreAutoProc.AutoProcContainer = autoProcContainer return inputStoreAutoProc
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, timeStart, timeEnd): if isAnom: anomString = "anom" else: anomString = "noanom" # Copy dataFiles to results directory for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: trunc, suffix = os.path.splitext(dataFile.path.value) newFileName = trunc + "_" + anomString + suffix shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName)) # Read the generated ISPyB xml file - if any if edPluginExecXia2DIALS.dataOutput.ispybXML is not None: autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration if isAnom: autoProcIntegration.anomalous = True else: autoProcIntegration.anomalous = False image = autoProcIntegrationContainer.Image image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = autoProcProgramContainer.AutoProcProgram autoProcProgram.processingPrograms = "XIA2_DIALS" autoProcProgram.processingStatus = True autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart) autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd) autoProcProgramContainer.AutoProcProgramAttachment = [] # Upload the log file to ISPyB if edPluginExecXia2DIALS.dataOutput.logFile is not None: pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log" shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the summary file to ISPyB if edPluginExecXia2DIALS.dataOutput.summary is not None: pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log" shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Create a pdf file of the html page if edPluginExecXia2DIALS.dataOutput.htmlFile is not None: pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.pdf" # Convert the xia2.html to xia2.pdf xsDataInputHTML2PDF = XSDataInputHTML2PDF() xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(pathToHtmlFile))) xsDataInputHTML2PDF.paperSize = XSDataString("A4") xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True) edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString)) edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF edPluginHTML2Pdf.executeSynchronous() pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Copy all log files for logFile in edPluginExecXia2DIALS.dataOutput.logFiles: pathToLogFile = logFile.path.value if pathToLogFile.endswith(".log"): pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile) shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Copy data files for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: pathToDataFile = dataFile.path.value if pathToDataFile.endswith(".mtz"): pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile) shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString)) edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoproc.executeSynchronous()
for iIndex, strArg in enumerate(sys.argv[1:]): strarg = strArg.lower() if strarg == "--verbose": EDVerbose.setVerboseOn() elif strarg == "--debug": EDVerbose.setVerboseDebugOn() strArgPath = strArg if os.path.dirname(strArgPath) == "": strArgPath = os.path.join(strCwd, strArg) strArgPathAbsolute = os.path.abspath(strArgPath) if os.path.exists(strArgPathAbsolute): listPaths.append(strArgPathAbsolute) if listPaths == []: EDVerbose.ERROR("No valid XML file given as input!") EDVerbose.ERROR("Arguments: %r" % sys.argv) EDVerbose.ERROR("Usage: edna-autoproc2ispyb path_to_xml_file [--debug]") sys.exit(1) for strPath in listPaths: EDVerbose.screen("Starting EDNA AutoProc -> ISPyB for file %s" % strPath) xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() strXMLAutoProcContainer = EDUtilsFile.readFile(strPath) xsDataAutoProcContainer = AutoProcContainer.parseString(strXMLAutoProcContainer) if xsDataAutoProcContainer is None: EDVerbose.WARNING("Couldn't parse file %s" % strPath) else: xsDataInputStoreAutoProc.setAutoProcContainer(xsDataAutoProcContainer) edPluginISPyBStoreAutoProc = EDFactoryPluginStatic.loadPlugin("EDPluginISPyBStoreAutoProcv1_4") edPluginISPyBStoreAutoProc.setDataInput(xsDataInputStoreAutoProc) edPluginISPyBStoreAutoProc.connectSUCCESS(successAction) edPluginISPyBStoreAutoProc.connectFAILURE(failureAction) edPluginISPyBStoreAutoProc.executeSynchronous()
strArgPath = strArg if os.path.dirname(strArgPath) == "": strArgPath = os.path.join(strCwd, strArg) strArgPathAbsolute = os.path.abspath(strArgPath) if os.path.exists(strArgPathAbsolute): listPaths.append(strArgPathAbsolute) if listPaths == []: EDVerbose.ERROR("No valid XML file given as input!") EDVerbose.ERROR("Arguments: %r" % sys.argv) EDVerbose.ERROR( "Usage: edna-autoproc2ispyb path_to_xml_file [--debug]") sys.exit(1) for strPath in listPaths: EDVerbose.screen("Starting EDNA AutoProc -> ISPyB for file %s" % strPath) xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() strXMLAutoProcContainer = EDUtilsFile.readFile(strPath) xsDataAutoProcContainer = AutoProcContainer.parseString( strXMLAutoProcContainer) if xsDataAutoProcContainer is None: EDVerbose.WARNING("Couldn't parse file %s" % strPath) else: if xsDataAutoProcContainer.AutoProcProgramContainer.AutoProcProgram.processingStatus == "True": xsDataAutoProcContainer.AutoProcProgramContainer.AutoProcProgram.processingStatus = "SUCCESS" xsDataInputStoreAutoProc.setAutoProcContainer( xsDataAutoProcContainer) edPluginISPyBStoreAutoProc = EDFactoryPluginStatic.loadPlugin( "EDPluginISPyBStoreAutoProcv1_4") edPluginISPyBStoreAutoProc.setDataInput(xsDataInputStoreAutoProc) edPluginISPyBStoreAutoProc.executeSynchronous() if edPluginISPyBStoreAutoProc.isFailure():
def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginControlCrystFELv1_0.postProcess") """ autoProcScalingStatisticsId : integer optional scalingStatisticsType : string comments : string resolutionLimitLow : float resolutionLimitHigh : float rMerge : float rMeasWithinIPlusIMinus : float rMeasAllIPlusIMinus : float rPimWithinIPlusIMinus : float rPimAllIPlusIMinus : float fractionalPartialBias : float nTotalObservations : integer ntotalUniqueObservations : integer meanIOverSigI : float completeness : float multiplicity : float anomalousCompleteness : float anomalousMultiplicity : float recordTimeStamp : string anomalous : boolean autoProcScalingId : integer ccHalf : float ccAno : float sigAno : float isa : float completenessSpherical : float anomalousCompletenessSpherical : float completenessEllipsoidal : float anomalousCompletenessEllipsoidal : float """ """ overallCompl : XSDataDouble overallRed : XSDataDouble overallSnr : XSDataDouble overallRsplit : XSDataDouble overallCC """ output = AutoProcContainer() autoproc = AutoProc() autoproc.spaceGroup = str(self.dataInput.spaceGroup.value) output.AutoProc = autoproc scaling_container = AutoProcScalingContainer() scaling = AutoProcScaling() scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) scaling_container.AutoProcScaling = scaling #Open the cell file and get the cell parameters cell_file = open(self.dataInput.cellFile.value, "r") cell_params = {} for line in cell_file.readlines(): if line.startswith("a ="): cell_params["a"] = float(line[4:9]) elif line.startswith("b ="): cell_params["b"] = float(line[4:9]) elif line.startswith("c ="): cell_params["c"] = float(line[4:9]) elif line.startswith("al ="): cell_params["alpha"] = float(line[4:9]) elif line.startswith("be ="): cell_params["beta"] = float(line[4:9]) elif line.startswith("ga ="): cell_params["gamma"] = float(line[4:9]) cell_file.close() autoproc.refinedCell_a = cell_params["a"] autoproc.refinedCell_b = cell_params["b"] autoproc.refinedCell_c = cell_params["c"] autoproc.refinedCell_alpha = cell_params["alpha"] autoproc.refinedCell_beta = cell_params["beta"] autoproc.refinedCell_gamma = cell_params["gamma"] overall_stats = AutoProcScalingStatistics() overall_stats.scalingStatisticsType = "innerShell" overall_stats.ccHalf = float( self.post_process_plugin.dataOutput.overallCC.value) overall_stats.resolutionLimitLow = float( self.post_process_plugin.dataOutput.resolutionLimitLow.value) overall_stats.resolutionLimitHigh = float( self.post_process_plugin.dataOutput.resolutionLimitHigh.value) overall_stats.completeness = float( self.post_process_plugin.dataOutput.overallCompl.value) #TODO get rMerge: otherwise results are not displayed in EXI overall_stats.rMerge = 0 scaling_container.AutoProcScalingStatistics.append(overall_stats) integration_container = AutoProcIntegrationContainer() image = Image() image.dataCollectionId = self.dataInput.dataCollectionId.value integration_container.Image = image integration = AutoProcIntegration() if self.integration_id is not None: integration.autoProcIntegrationId = self.integration_id integration.cell_a = cell_params["a"] integration.cell_b = cell_params["b"] integration.cell_c = cell_params["c"] integration.cell_alpha = cell_params["alpha"] integration.cell_beta = cell_params["beta"] integration.cell_gamma = cell_params["gamma"] integration.anomalous = 0 # done with the integration integration_container.AutoProcIntegration = integration scaling_container.AutoProcIntegrationContainer = integration_container program_container = AutoProcProgramContainer() program_container.AutoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=self.program_id, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) output.AutoProcProgramContainer = program_container output.AutoProcScalingContainer = scaling_container for log_file in self.post_process_plugin.dataOutput.logFiles: pathToLogFile = log_file.path.value pyarchFileName = os.path.join(self.pyarchDirectory, os.path.basename(pathToLogFile)) #shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoproc_program_attachment = AutoProcProgramAttachment() autoproc_program_attachment.fileName = os.path.basename( pyarchFileName) autoproc_program_attachment.filePath = os.path.dirname( pyarchFileName) autoproc_program_attachment.fileType = "Log" program_container.addAutoProcProgramAttachment( autoproc_program_attachment) self.screen("Result file %s uploaded to ISPyB" % os.path.basename(pyarchFileName)) for data_file in self.post_process_plugin.dataOutput.dataFiles: pathToDataFile = data_file.path.value pyarchFileName = os.path.join(self.pyarchDirectory, os.path.basename(pathToDataFile)) #shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoproc_program_attachment = AutoProcProgramAttachment() autoproc_program_attachment.fileName = os.path.basename( pyarchFileName) autoproc_program_attachment.filePath = os.path.dirname( pyarchFileName) autoproc_program_attachment.fileType = "Result" program_container.addAutoProcProgramAttachment( autoproc_program_attachment) self.screen("Result file %s uploaded to ISPyB" % os.path.basename(pyarchFileName)) ispyb_input = XSDataInputStoreAutoProc() ispyb_input.AutoProcContainer = output self.store_autoproc_plugin.dataInput = ispyb_input self.store_autoproc_plugin.executeSynchronous() if self.store_autoproc_plugin.isFailure(): self.ERROR("Could not upload results to ispyb!") else: self.hasUploadedAnomResultsToISPyB = True self.screen("Results uploaded to ISPyB") xsDataInput = XSDataInputISPyBUpdateDataCollectionGroupComment() xsDataInput.newComment = XSDataString( self.post_process_plugin.dataOutput.comment.value) xsDataInput.dataCollectionId = self.dataInput.dataCollectionId self.store_dc_comment_plugin.dataInput = xsDataInput self.executePluginSynchronous(self.store_dc_comment_plugin)
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template, strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId, integrationId=None, programId=None): # Parse log file dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value) dictXscale = self.parseXscaleLp(strPathXscaleLp) xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() autoProcContainer = AutoProcContainer() # AutoProc autoProc = AutoProc() autoProc.spaceGroup = dictLog["spaceGroup"] autoProc.refinedCell_a = dictLog["cellA"] autoProc.refinedCell_b = dictLog["cellB"] autoProc.refinedCell_c = dictLog["cellC"] autoProc.refinedCell_alpha = dictLog["cellAlpha"] autoProc.refinedCell_beta = dictLog["cellBeta"] autoProc.refinedCell_gamma = dictLog["cellGamma"] autoProcContainer.AutoProc = autoProc # AutoProcIntegrationContainer autoProcIntegrationContainer = AutoProcIntegrationContainer() autoProcIntegration = AutoProcIntegration() autoProcIntegration.autoProcIntegrationId = integrationId autoProcIntegration.cell_a = dictLog["cellA"] autoProcIntegration.cell_b = dictLog["cellB"] autoProcIntegration.cell_c = dictLog["cellC"] autoProcIntegration.cell_alpha = dictLog["cellAlpha"] autoProcIntegration.cell_beta = dictLog["cellBeta"] autoProcIntegration.cell_gamma = dictLog["cellGamma"] autoProcIntegration.anomalous = isAnom image = Image() image.dataCollectionId = dataCollectionId autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration autoProcIntegrationContainer.Image = image # Scaling container if xsDataResultXDSAPP.correctLP is not None: isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value) else: isa = None autoProcScalingContainer = AutoProcScalingContainer() autoProcScaling = AutoProcScaling() autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) autoProcScalingContainer.AutoProcScaling = autoProcScaling for scalingStatisticsType in dictXscale: autoProcScalingStatistics = AutoProcScalingStatistics() autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType autoProcScalingStatistics.anomalous = isAnom for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]: setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute]) if scalingStatisticsType == "overall" and isa is not None: autoProcScalingStatistics.isa = isa autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics) autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer # Program autoProcProgramContainer = AutoProcProgramContainer() autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram # XDSAPP log and result files if xsDataResultXDSAPP.logFile is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value, "xdsapp", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.pointlessLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value, "pointless", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.phenixXtriageLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value, "xtriage", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.correctLP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value, "CORRECT", "LP", isAnom, attachmentType="Log") if xsDataResultXDSAPP.XDS_ASCII_HKL is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value, "XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True) if xsDataResultXDSAPP.XDS_INP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value, "XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True) for mtz_F in xsDataResultXDSAPP.mtz_F: basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F.path.value, basenameMtz_F, "mtz", isAnom, attachmentType="Result") for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus: basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value, basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result") # for mtz_I in xsDataResultXDSAPP.mtz_I: # basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0] # self.addAttachment(autoProcProgramContainer, mtz_I.path.value, # basenameMtz_I, "mtz", isAnom, attachmentType="Result") # for hkl in xsDataResultXDSAPP.hkl: # basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0] # self.addAttachment(autoProcProgramContainer, hkl.path.value, # basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True) # for cv in xsDataResultXDSAPP.cv: # basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0] # self.addAttachment(autoProcProgramContainer, cv.path.value, # basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True) if os.path.exists(strPathXscaleLp): self.addAttachment(autoProcProgramContainer, strPathXscaleLp, "XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result") autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer return xsDataInputStoreAutoProc
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId, integrationId): successUpload = False if isAnom: anomString = "anom" else: anomString = "noanom" # Read the generated ISPyB xml file - if any if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None: autoProcContainer = AutoProcContainer.parseFile( edPluginExecXia2DIALS.dataOutput.ispybXML.path.value) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration autoProcIntegration.autoProcIntegrationId = integrationId if isAnom: autoProcIntegration.anomalous = True else: autoProcIntegration.anomalous = False image = autoProcIntegrationContainer.Image image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram autoProcProgramContainer.AutoProcProgramAttachment = [] # Upload the log file to ISPyB if edPluginExecXia2DIALS.dataOutput.logFile is not None: pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log" shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Upload the summary file to ISPyB if edPluginExecXia2DIALS.dataOutput.summary is not None: pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log" shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToSummaryFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Create a pdf file of the html page if edPluginExecXia2DIALS.dataOutput.htmlFile is not None: pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html" shutil.copy(pathToHtmlFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToHtmlFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Copy all log files for logFile in edPluginExecXia2DIALS.dataOutput.logFiles: pathToLogFile = logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename( pathToLogFile) # Copy all log files to results: shutil.copy( pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) # Only copy .log files to pyarch if pathToLogFile.endswith(".log"): shutil.copy( pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Copy data files for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: pathToDataFile = dataFile.path.value if pathToDataFile.endswith(".mtz"): pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename( pathToDataFile) shutil.copy( pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy( pathToDataFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment( autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoproc = self.loadPlugin( "EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString)) edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoproc.executeSynchronous() successUpload = not edPluginStoreAutoproc.isFailure() else: # Copy dataFiles to results directory for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: trunc, suffix = os.path.splitext(dataFile.path.value) newFileName = os.path.basename( trunc) + "_" + anomString + suffix shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName)) return successUpload
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template, strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId, integrationId=None, programId=None): # Parse log file dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value) dictXscale = self.parseXscaleLp(strPathXscaleLp) # Create path to pyarch self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory) self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA') if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory): os.makedirs(self.pyarchDirectory, 0o755) # Determine pyarch prefix listPrefix = template.split("_") self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2]) xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() autoProcContainer = AutoProcContainer() # AutoProc autoProc = AutoProc() autoProc.spaceGroup = dictLog["spaceGroup"] autoProc.refinedCell_a = dictLog["cellA"] autoProc.refinedCell_b = dictLog["cellB"] autoProc.refinedCell_c = dictLog["cellC"] autoProc.refinedCell_alpha = dictLog["cellAlpha"] autoProc.refinedCell_beta = dictLog["cellBeta"] autoProc.refinedCell_gamma = dictLog["cellGamma"] autoProcContainer.AutoProc = autoProc # AutoProcIntegrationContainer autoProcIntegrationContainer = AutoProcIntegrationContainer() autoProcIntegration = AutoProcIntegration() autoProcIntegration.autoProcIntegrationId = integrationId autoProcIntegration.cell_a = dictLog["cellA"] autoProcIntegration.cell_b = dictLog["cellB"] autoProcIntegration.cell_c = dictLog["cellC"] autoProcIntegration.cell_alpha = dictLog["cellAlpha"] autoProcIntegration.cell_beta = dictLog["cellBeta"] autoProcIntegration.cell_gamma = dictLog["cellGamma"] autoProcIntegration.anomalous = isAnom image = Image() image.dataCollectionId = dataCollectionId autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration autoProcIntegrationContainer.Image = image # Scaling container if xsDataResultXDSAPP.correctLP is not None: isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value) else: isa = None autoProcScalingContainer = AutoProcScalingContainer() autoProcScaling = AutoProcScaling() autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) autoProcScalingContainer.AutoProcScaling = autoProcScaling for scalingStatisticsType in dictXscale: autoProcScalingStatistics = AutoProcScalingStatistics() autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType autoProcScalingStatistics.anomalous = isAnom for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]: setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute]) if scalingStatisticsType == "overall" and isa is not None: autoProcScalingStatistics.isa = isa autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics) autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer # Program autoProcProgramContainer = AutoProcProgramContainer() autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram # Attached files pyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(processDirectory) pyarchResultPath = os.path.join(pyarchPath, "results") if not os.path.exists(pyarchResultPath): os.makedirs(pyarchResultPath, 0o755) # XDSAPP log and result files if xsDataResultXDSAPP.logFile is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value, "xdsapp", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.pointlessLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value, "pointless", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.phenixXtriageLog is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value, "xtriage", "log", isAnom, attachmentType="Log") if xsDataResultXDSAPP.correctLP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value, "CORRECT", "LP", isAnom, attachmentType="Log") if xsDataResultXDSAPP.XDS_ASCII_HKL is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value, "XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True) if xsDataResultXDSAPP.XDS_INP is not None: self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value, "XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True) for mtz_F in xsDataResultXDSAPP.mtz_F: basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F.path.value, basenameMtz_F, "mtz", isAnom, attachmentType="Result") for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus: basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0] self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value, basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result") # for mtz_I in xsDataResultXDSAPP.mtz_I: # basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0] # self.addAttachment(autoProcProgramContainer, mtz_I.path.value, # basenameMtz_I, "mtz", isAnom, attachmentType="Result") # for hkl in xsDataResultXDSAPP.hkl: # basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0] # self.addAttachment(autoProcProgramContainer, hkl.path.value, # basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True) # for cv in xsDataResultXDSAPP.cv: # basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0] # self.addAttachment(autoProcProgramContainer, cv.path.value, # basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True) if os.path.exists(strPathXscaleLp): self.addAttachment(autoProcProgramContainer, strPathXscaleLp, "XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result") autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer return xsDataInputStoreAutoProc
def uploadToISPyB(self, edPluginExecXia2DIALS, isAnom, proposal, programId, integrationId): successUpload = False if isAnom: anomString = "anom" else: anomString = "noanom" # Read the generated ISPyB xml file - if any if self.pyarchDirectory is not None and edPluginExecXia2DIALS.dataOutput.ispybXML is not None: autoProcContainer = AutoProcContainer.parseFile(edPluginExecXia2DIALS.dataOutput.ispybXML.path.value) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration autoProcIntegration.autoProcIntegrationId = integrationId if isAnom: autoProcIntegration.anomalous = True else: autoProcIntegration.anomalous = False image = autoProcIntegrationContainer.Image image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram( programId=programId, status="SUCCESS", timeStart=self.timeStart, timeEnd=self.timeEnd, processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms) autoProcProgramContainer.AutoProcProgram = autoProcProgram autoProcProgramContainer.AutoProcProgramAttachment = [] # Upload the log file to ISPyB if edPluginExecXia2DIALS.dataOutput.logFile is not None: pathToLogFile = edPluginExecXia2DIALS.dataOutput.logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.log" shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the summary file to ISPyB if edPluginExecXia2DIALS.dataOutput.summary is not None: pathToSummaryFile = edPluginExecXia2DIALS.dataOutput.summary.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2-summary.log" shutil.copy(pathToSummaryFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToSummaryFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Create a pdf file of the html page if edPluginExecXia2DIALS.dataOutput.htmlFile is not None: pathToHtmlFile = edPluginExecXia2DIALS.dataOutput.htmlFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_xia2.html" shutil.copy(pathToHtmlFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToHtmlFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Copy all log files for logFile in edPluginExecXia2DIALS.dataOutput.logFiles: pathToLogFile = logFile.path.value pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToLogFile) # Copy all log files to results: shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchFileName)) # Only copy .log files to pyarch if pathToLogFile.endswith(".log"): shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Copy data files for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: pathToDataFile = dataFile.path.value if pathToDataFile.endswith(".mtz"): pyarchFileName = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pathToDataFile) shutil.copy(pathToDataFile, os.path.join(self.pyarchDirectory, pyarchFileName)) shutil.copy(pathToDataFile, os.path.join(self.resultsDirectory, pyarchFileName)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchFileName autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoproc = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "EDPluginISPyBStoreAutoProcv1_4_{0}".format(anomString)) edPluginStoreAutoproc.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoproc.executeSynchronous() successUpload = not edPluginStoreAutoproc.isFailure() else: # Copy dataFiles to results directory for dataFile in edPluginExecXia2DIALS.dataOutput.dataFiles: trunc, suffix = os.path.splitext(dataFile.path.value) newFileName = os.path.basename(trunc) + "_" + anomString + suffix shutil.copy(dataFile.path.value, os.path.join(self.resultsDirectory, newFileName)) return successUpload
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, proposal, timeStart, timeEnd): if isAnom: anomString = "anom" else: anomString = "noanom" # Read the generated ISPyB xml file - if any if edPluginExecAutoPROC.dataOutput.ispybXML is not None: autoProcContainer = AutoProcContainer.parseFile(edPluginExecAutoPROC.dataOutput.ispybXML.path.value) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer image = autoProcIntegrationContainer.Image image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration if isAnom: autoProcIntegration.anomalous = True else: autoProcIntegration.anomalous = False autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = autoProcProgramContainer.AutoProcProgram autoProcProgram.processingPrograms = "autoPROC" autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart) autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd) for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment: if autoProcProgramAttachment.fileName == "summary.html": summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) # Replace opidXX with user name htmlSummary = open(summaryHtmlPath).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) htmlSummary = htmlSummary.replace(userString1, userString2) open(summaryHtmlPath, "w").write(htmlSummary) # Convert the summary.html to summary.pdf xsDataInputHTML2PDF = XSDataInputHTML2PDF() xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath))) xsDataInputHTML2PDF.paperSize = XSDataString("A3") xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True) edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString)) edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF edPluginHTML2Pdf.executeSynchronous() pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile) # Copy file to results directory and pyarch shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile)) shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile)) autoProcProgramAttachment.fileName = pyarchPdfFile autoProcProgramAttachment.filePath = self.pyarchDirectory elif autoProcProgramAttachment.fileName == "truncate-unique.mtz": pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString) shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory else: pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory # Add XSCALE.LP file if present processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value pathToXSCALELog = os.path.join(processDirectory, "xscale_XSCALE.LP") if os.path.exists(pathToXSCALELog): pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString) shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog)) shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXSCALELog autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add XDS_ASCII.HKL if present and gzip it pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL") if os.path.exists(pathToXdsAsciiHkl): pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString) f_in = open(pathToXdsAsciiHkl) f_out = gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), "wb") f_out.writelines(f_in) f_out.close() f_in.close() shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add log file pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value autoPROClog = open(pathToLogFile).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) autoPROClog = autoPROClog.replace(userString1, userString2) open(pathToLogFile, "w").write(autoPROClog) pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString) shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile)) shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchLogFile autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer if isAnom: self.edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc self.edPluginStoreAutoprocAnom.executeSynchronous() else: self.edPluginStoreAutoprocNoanom.dataInput = xsDataInputStoreAutoProc self.edPluginStoreAutoprocNoanom.executeSynchronous()
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, isStaraniso, proposal, timeStart, timeEnd): if isAnom: anomString = "anom" else: anomString = "noanom" if isStaraniso: staranisoString = "_staraniso" else: staranisoString = "" # Read the generated ISPyB xml file pathToISPyBXML = None if isStaraniso: if edPluginExecAutoPROC.dataOutput.ispybXML_staraniso is not None: pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML_staraniso.path.value elif edPluginExecAutoPROC.dataOutput.ispybXML is not None: pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML.path.value if pathToISPyBXML is not None: autoProcContainer = AutoProcContainer.parseFile(pathToISPyBXML) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcScalingStatistics.rMeasWithinIPlusIMinus *= 100 autoProcScalingStatistics.rMeasAllIPlusIMinus *= 100 autoProcScalingStatistics.rPimWithinIPlusIMinus *= 100 autoProcScalingStatistics.rPimAllIPlusIMinus *= 100 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer image = autoProcIntegrationContainer.Image if self.dataInput.dataCollectionId is not None: image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = autoProcProgramContainer.AutoProcProgram if isAnom: autoProcIntegration.anomalous = True if isStaraniso: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnomStaraniso autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnomStaraniso else: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnom autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnom else: autoProcIntegration.anomalous = False if isStaraniso: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanomStaraniso autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanomStaraniso else: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanom autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanom if self.reprocess: autoProcProgram.processingPrograms = "autoPROC" + staranisoString + " reprocess" else: autoProcProgram.processingPrograms = "autoPROC" + staranisoString autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart) autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd) autoProcProgram.processingStatus = "SUCCESS" # EDNA-245 - remove "truncate_{early,late}-unique.mtz" from autoProcProgramContainer.AutoProcProgramAttachment autoProcProgramContainer.AutoProcProgramAttachment[:] = [x for x in autoProcProgramContainer.AutoProcProgramAttachment if not self.matchesTruncateEarlyLate(x.fileName) ] for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment: if autoProcProgramAttachment.fileName == "summary.html": # Check if summary_inlined.html exists summaryInlinedHtmlPath = os.path.join(autoProcProgramAttachment.filePath, "summary_inlined.html") if os.path.exists(summaryInlinedHtmlPath): summaryName = "summary_inlined" summaryHtmlPath = summaryInlinedHtmlPath else: summaryName = "summary" summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) # Replace opidXX with user name htmlSummary = open(summaryHtmlPath).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) htmlSummary = htmlSummary.replace(userString1, userString2) open(summaryHtmlPath, "w").write(htmlSummary) # Upload summary.html pathtoFile = summaryHtmlPath pyarchFile = self.pyarchPrefix + "_{0}_{1}.html".format(anomString, summaryName) if not pyarchFile in self.listPyarchFile: if self.resultsDirectory: shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) self.listPyarchFile.append(pyarchFile) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = os.path.basename(pyarchFile) autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" if summaryName == "summary": # Convert the summary.html to summary.pdf xsDataInputHTML2PDF = XSDataInputHTML2PDF() xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath))) xsDataInputHTML2PDF.paperSize = XSDataString("A3") xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True) edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString)) edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF edPluginHTML2Pdf.executeSynchronous() pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile) # Copy file to results directory and pyarch if self.resultsDirectory: shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile)) if self.pyarchDirectory is not None: shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile)) autoProcProgramAttachmentPdf = AutoProcProgramAttachment() autoProcProgramAttachmentPdf.fileName = pyarchPdfFile autoProcProgramAttachmentPdf.filePath = self.pyarchDirectory autoProcProgramAttachmentPdf.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachmentPdf) elif autoProcProgramAttachment.fileName == "truncate-unique.mtz": pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString) if self.resultsDirectory: shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory else: pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName if self.resultsDirectory: shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory # Add XSCALE.LP file if present processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value pathToXSCALELog = os.path.join(processDirectory, "XSCALE.LP") if os.path.exists(pathToXSCALELog): pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString) if self.resultsDirectory: shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog)) if self.pyarchDirectory is not None: shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXSCALELog autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add XDS_ASCII.HKL if present and gzip it pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL") if os.path.exists(pathToXdsAsciiHkl) and self.pyarchDirectory is not None: pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString) with open(pathToXdsAsciiHkl, 'rb') as f_in: with gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), 'wb') as f_out: shutil.copyfileobj(f_in, f_out) if self.resultsDirectory: shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add log file pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value autoPROClog = open(pathToLogFile).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) autoPROClog = autoPROClog.replace(userString1, userString2) open(pathToLogFile, "w").write(autoPROClog) pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString) if self.resultsDirectory: shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile)) if self.pyarchDirectory is not None: shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchLogFile autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add report.pdf pathToRepordPdf = None if isStaraniso and edPluginExecAutoPROC.dataOutput.reportPdf_staraniso is not None: pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf_staraniso.path.value elif edPluginExecAutoPROC.dataOutput.reportPdf is not None: pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf.path.value if pathToRepordPdf is not None: pyarchReportFile = self.pyarchPrefix + "_{0}_{1}".format(anomString, os.path.basename(pathToRepordPdf)) if self.resultsDirectory: shutil.copy(pathToRepordPdf, os.path.join(self.resultsDirectory, pyarchReportFile)) if self.pyarchDirectory is not None: shutil.copy(pathToRepordPdf, os.path.join(self.pyarchDirectory, pyarchReportFile)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchReportFile autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoprocAnom = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "ISPyBStoreAutoProcv1_4_{0}{1}".format(anomString, staranisoString)) edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoprocAnom.executeSynchronous() isSuccess = not edPluginStoreAutoprocAnom.isFailure() if isSuccess: self.screen("{0}{1} results uploaded to ISPyB".format(anomString, staranisoString)) if isAnom: if isStaraniso: self.hasUploadedAnomStaranisoResultsToISPyB = True else: self.hasUploadedAnomResultsToISPyB = True else: if isStaraniso: self.hasUploadedNoanomStaranisoResultsToISPyB = True else: self.hasUploadedNoanomResultsToISPyB = True else: self.screen("Could not upload {0}{1} results to ISPyB".format(anomString, staranisoString))
def postProcess(self, _edObject = None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginControlAutoproc.postProcess") #Now that we have executed the whole thing we need to create #the suitable ISPyB plugin input and serialize it to the file #we've been given as input output = AutoProcContainer() # AutoProc attr autoproc = AutoProc() xdsout = self.xds_first.dataOutput if xdsout.sg_number is not None: # and it should not autoproc.spaceGroup = SPACE_GROUP_NAMES[xdsout.sg_number.value] autoproc.refinedCell_a = xdsout.cell_a.value autoproc.refinedCell_b = xdsout.cell_b.value autoproc.refinedCell_c = xdsout.cell_c.value autoproc.refinedCell_alpha = xdsout.cell_alpha.value autoproc.refinedCell_beta = xdsout.cell_beta.value autoproc.refinedCell_gamma = xdsout.cell_gamma.value output.AutoProc = autoproc # scaling container and all the things that go in scaling_container_noanom = AutoProcScalingContainer() scaling = AutoProcScaling() scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) scaling_container_noanom.AutoProcScaling = scaling # NOANOM PATH xscale_stats_noanom = self.xscale_generate.dataOutput.stats_noanom_merged inner_stats_noanom = xscale_stats_noanom.completeness_entries[0] outer_stats_noanom = xscale_stats_noanom.completeness_entries[-1] # use the previous shell's res as low res, if available prev_res = self.low_resolution_limit try: prev_res = xscale_stats_noanom.completeness_entries[-2].outer_res.value except IndexError: pass total_stats_noanom = xscale_stats_noanom.total_completeness stats = _create_scaling_stats(inner_stats_noanom, 'innerShell', self.low_resolution_limit, False) overall_low = stats.resolutionLimitLow scaling_container_noanom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(outer_stats_noanom, 'outerShell', prev_res, False) overall_high = stats.resolutionLimitHigh scaling_container_noanom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(total_stats_noanom, 'overall', self.low_resolution_limit, False) stats.resolutionLimitLow = overall_low stats.resolutionLimitHigh = overall_high scaling_container_noanom.AutoProcScalingStatistics.append(stats) integration_container_noanom = AutoProcIntegrationContainer() image = Image() image.dataCollectionId = self.dataInput.data_collection_id.value integration_container_noanom.Image = image integration_noanom = AutoProcIntegration() if self.integration_id_noanom is not None: integration_noanom.autoProcIntegrationId = self.integration_id_noanom crystal_stats = self.parse_xds_noanom.dataOutput integration_noanom.cell_a = crystal_stats.cell_a.value integration_noanom.cell_b = crystal_stats.cell_b.value integration_noanom.cell_c = crystal_stats.cell_c.value integration_noanom.cell_alpha = crystal_stats.cell_alpha.value integration_noanom.cell_beta = crystal_stats.cell_beta.value integration_noanom.cell_gamma = crystal_stats.cell_gamma.value integration_noanom.anomalous = 0 # done with the integration integration_container_noanom.AutoProcIntegration = integration_noanom scaling_container_noanom.AutoProcIntegrationContainer = integration_container_noanom # ANOM PATH scaling_container_anom = AutoProcScalingContainer() scaling = AutoProcScaling() scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) scaling_container_anom.AutoProcScaling = scaling xscale_stats_anom = self.xscale_generate.dataOutput.stats_anom_merged inner_stats_anom = xscale_stats_anom.completeness_entries[0] outer_stats_anom = xscale_stats_anom.completeness_entries[-1] # use the previous shell's res as low res if available prev_res = self.low_resolution_limit try: prev_res = xscale_stats_anom.completeness_entries[-2].outer_res.value except IndexError: pass total_stats_anom = xscale_stats_anom.total_completeness stats = _create_scaling_stats(inner_stats_anom, 'innerShell', self.low_resolution_limit, True) overall_low = stats.resolutionLimitLow scaling_container_anom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(outer_stats_anom, 'outerShell', prev_res, True) overall_high = stats.resolutionLimitHigh scaling_container_anom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(total_stats_anom, 'overall', self.low_resolution_limit, True) stats.resolutionLimitLow = overall_low stats.resolutionLimitHigh = overall_high scaling_container_anom.AutoProcScalingStatistics.append(stats) integration_container_anom = AutoProcIntegrationContainer() image = Image() image.dataCollectionId = self.dataInput.data_collection_id.value integration_container_anom.Image = image integration_anom = AutoProcIntegration() crystal_stats = self.parse_xds_anom.dataOutput if self.integration_id_anom is not None: integration_anom.autoProcIntegrationId = self.integration_id_anom integration_anom.cell_a = crystal_stats.cell_a.value integration_anom.cell_b = crystal_stats.cell_b.value integration_anom.cell_c = crystal_stats.cell_c.value integration_anom.cell_alpha = crystal_stats.cell_alpha.value integration_anom.cell_beta = crystal_stats.cell_beta.value integration_anom.cell_gamma = crystal_stats.cell_gamma.value integration_anom.anomalous = 1 # done with the integration integration_container_anom.AutoProcIntegration = integration_anom scaling_container_anom.AutoProcIntegrationContainer = integration_container_anom # ------ NO ANOM / ANOM end program_container = AutoProcProgramContainer() program_container.AutoProcProgram = AutoProcProgram() program_container.AutoProcProgram.processingCommandLine = ' '.join(sys.argv) program_container.AutoProcProgram.processingPrograms = 'edna-fastproc' # now for the generated files. There's some magic to do with # their paths to determine where to put them on pyarch pyarch_path = None # Note: the path is in the form /data/whatever # remove the edna-autoproc-import suffix original_files_dir = self.file_conversion.dataInput.output_directory.value #files_dir, _ = os.path.split(original_files_dir) files_dir = original_files_dir # the whole transformation is fragile! if files_dir.startswith('/data/visitor'): # We might get empty elements at the head/tail of the list tokens = [elem for elem in files_dir.split(os.path.sep) if len(elem) > 0] pyarch_path = os.path.join('/data/pyarch', tokens[3], tokens[2], *tokens[4:]) else: # We might get empty elements at the head/tail of the list tokens = [elem for elem in files_dir.split(os.path.sep) if len(elem) > 0] if tokens[2] == 'inhouse': pyarch_path = os.path.join('/data/pyarch', tokens[1], *tokens[3:]) if pyarch_path is not None: pyarch_path = pyarch_path.replace('PROCESSED_DATA', 'RAW_DATA') try: os.makedirs(pyarch_path) except OSError: # dir already exists, may happen when testing EDVerbose.screen('Target directory on pyarch ({0}) already exists, ignoring'.format(pyarch_path)) file_list = [] # we can now copy the files to this dir for f in os.listdir(original_files_dir): current = os.path.join(original_files_dir, f) if not os.path.isfile(current): continue if not os.path.splitext(current)[1].lower() in ISPYB_UPLOAD_EXTENSIONS: continue new_path = os.path.join(pyarch_path, f) file_list.append(new_path) shutil.copyfile(current, new_path) # now add those to the ispyb upload for path in file_list: dirname, filename = os.path.split(path) attach = AutoProcProgramAttachment() attach.fileType = "Result" attach.fileName = filename attach.filePath = dirname program_container.AutoProcProgramAttachment.append(attach) program_container.AutoProcProgram.processingStatus = True output.AutoProcProgramContainer = program_container # first with anom output.AutoProcScalingContainer = scaling_container_anom ispyb_input = XSDataInputStoreAutoProc() ispyb_input.AutoProcContainer = output with open(self.dataInput.output_file.path.value, 'w') as f: f.write(ispyb_input.marshal()) # store results in ispyb self.store_autoproc_anom.dataInput = ispyb_input t0=time.time() self.store_autoproc_anom.executeSynchronous() self.stats['ispyb_upload'] = time.time() - t0 with open(self.log_file_path, 'w') as f: json.dump(self.stats, f) if self.store_autoproc_anom.isFailure(): self.ERROR('could not send results to ispyb') else: # store the autoproc ID as a filename in the # fastproc_integration_ids directory os.mknod(os.path.join(self.autoproc_ids_dir, str(self.integration_id_anom)), 0755) # then noanom stats output.AutoProcScalingContainer = scaling_container_noanom ispyb_input = XSDataInputStoreAutoProc() ispyb_input.AutoProcContainer = output with open(self.dataInput.output_file.path.value, 'w') as f: f.write(ispyb_input.marshal()) # store results in ispyb self.store_autoproc_noanom.dataInput = ispyb_input t0=time.time() self.store_autoproc_noanom.executeSynchronous() self.stats['ispyb_upload'] = time.time() - t0 with open(self.log_file_path, 'w') as f: json.dump(self.stats, f) if self.store_autoproc_noanom.isFailure(): self.ERROR('could not send results to ispyb') else: # store the autoproc id os.mknod(os.path.join(self.autoproc_ids_dir, str(self.integration_id_noanom)), 0755)
def postProcess(self, _edObject=None): EDPluginControl.postProcess(self) self.DEBUG("EDPluginControlAutoproc.postProcess") #Now that we have executed the whole thing we need to create #the suitable ISPyB plugin input and serialize it to the file #we've been given as input output = AutoProcContainer() # AutoProc attr autoproc = AutoProc() xdsout = self.xds_first.dataOutput if xdsout.sg_number is not None: # and it should not autoproc.spaceGroup = SPACE_GROUP_NAMES[xdsout.sg_number.value] autoproc.refinedCell_a = xdsout.cell_a.value autoproc.refinedCell_b = xdsout.cell_b.value autoproc.refinedCell_c = xdsout.cell_c.value autoproc.refinedCell_alpha = xdsout.cell_alpha.value autoproc.refinedCell_beta = xdsout.cell_beta.value autoproc.refinedCell_gamma = xdsout.cell_gamma.value output.AutoProc = autoproc # scaling container and all the things that go in scaling_container_noanom = AutoProcScalingContainer() scaling = AutoProcScaling() scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) scaling_container_noanom.AutoProcScaling = scaling # NOANOM PATH xscale_stats_noanom = self.xscale_generate.dataOutput.stats_noanom_merged inner_stats_noanom = xscale_stats_noanom.completeness_entries[0] outer_stats_noanom = xscale_stats_noanom.completeness_entries[-1] # use the previous shell's res as low res, if available prev_res = self.low_resolution_limit try: prev_res = xscale_stats_noanom.completeness_entries[ -2].outer_res.value except IndexError: pass total_stats_noanom = xscale_stats_noanom.total_completeness stats = _create_scaling_stats(inner_stats_noanom, 'innerShell', self.low_resolution_limit, False) overall_low = stats.resolutionLimitLow scaling_container_noanom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(outer_stats_noanom, 'outerShell', prev_res, False) overall_high = stats.resolutionLimitHigh scaling_container_noanom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(total_stats_noanom, 'overall', self.low_resolution_limit, False) stats.resolutionLimitLow = overall_low stats.resolutionLimitHigh = overall_high scaling_container_noanom.AutoProcScalingStatistics.append(stats) integration_container_noanom = AutoProcIntegrationContainer() image = Image() image.dataCollectionId = self.dataInput.data_collection_id.value integration_container_noanom.Image = image integration_noanom = AutoProcIntegration() if self.integration_id_noanom is not None: integration_noanom.autoProcIntegrationId = self.integration_id_noanom crystal_stats = self.parse_xds_noanom.dataOutput integration_noanom.cell_a = crystal_stats.cell_a.value integration_noanom.cell_b = crystal_stats.cell_b.value integration_noanom.cell_c = crystal_stats.cell_c.value integration_noanom.cell_alpha = crystal_stats.cell_alpha.value integration_noanom.cell_beta = crystal_stats.cell_beta.value integration_noanom.cell_gamma = crystal_stats.cell_gamma.value integration_noanom.anomalous = 0 # done with the integration integration_container_noanom.AutoProcIntegration = integration_noanom scaling_container_noanom.AutoProcIntegrationContainer = integration_container_noanom # ANOM PATH scaling_container_anom = AutoProcScalingContainer() scaling = AutoProcScaling() scaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) scaling_container_anom.AutoProcScaling = scaling xscale_stats_anom = self.xscale_generate.dataOutput.stats_anom_merged inner_stats_anom = xscale_stats_anom.completeness_entries[0] outer_stats_anom = xscale_stats_anom.completeness_entries[-1] # use the previous shell's res as low res if available prev_res = self.low_resolution_limit try: prev_res = xscale_stats_anom.completeness_entries[ -2].outer_res.value except IndexError: pass total_stats_anom = xscale_stats_anom.total_completeness stats = _create_scaling_stats(inner_stats_anom, 'innerShell', self.low_resolution_limit, True) overall_low = stats.resolutionLimitLow scaling_container_anom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(outer_stats_anom, 'outerShell', prev_res, True) overall_high = stats.resolutionLimitHigh scaling_container_anom.AutoProcScalingStatistics.append(stats) stats = _create_scaling_stats(total_stats_anom, 'overall', self.low_resolution_limit, True) stats.resolutionLimitLow = overall_low stats.resolutionLimitHigh = overall_high scaling_container_anom.AutoProcScalingStatistics.append(stats) integration_container_anom = AutoProcIntegrationContainer() image = Image() image.dataCollectionId = self.dataInput.data_collection_id.value integration_container_anom.Image = image integration_anom = AutoProcIntegration() crystal_stats = self.parse_xds_anom.dataOutput if self.integration_id_anom is not None: integration_anom.autoProcIntegrationId = self.integration_id_anom integration_anom.cell_a = crystal_stats.cell_a.value integration_anom.cell_b = crystal_stats.cell_b.value integration_anom.cell_c = crystal_stats.cell_c.value integration_anom.cell_alpha = crystal_stats.cell_alpha.value integration_anom.cell_beta = crystal_stats.cell_beta.value integration_anom.cell_gamma = crystal_stats.cell_gamma.value integration_anom.anomalous = 1 # done with the integration integration_container_anom.AutoProcIntegration = integration_anom scaling_container_anom.AutoProcIntegrationContainer = integration_container_anom # ------ NO ANOM / ANOM end program_container = AutoProcProgramContainer() program_container.AutoProcProgram = AutoProcProgram() program_container.AutoProcProgram.processingCommandLine = ' '.join( sys.argv) program_container.AutoProcProgram.processingPrograms = 'edna-fastproc' # now for the generated files. There's some magic to do with # their paths to determine where to put them on pyarch pyarch_path = None # Note: the path is in the form /data/whatever # remove the edna-autoproc-import suffix original_files_dir = self.file_conversion.dataInput.output_directory.value #files_dir, _ = os.path.split(original_files_dir) files_dir = original_files_dir # the whole transformation is fragile! if files_dir.startswith('/data/visitor'): # We might get empty elements at the head/tail of the list tokens = [ elem for elem in files_dir.split(os.path.sep) if len(elem) > 0 ] pyarch_path = os.path.join('/data/pyarch', tokens[3], tokens[2], *tokens[4:]) else: # We might get empty elements at the head/tail of the list tokens = [ elem for elem in files_dir.split(os.path.sep) if len(elem) > 0 ] if tokens[2] == 'inhouse': pyarch_path = os.path.join('/data/pyarch', tokens[1], *tokens[3:]) if pyarch_path is not None: pyarch_path = pyarch_path.replace('PROCESSED_DATA', 'RAW_DATA') try: os.makedirs(pyarch_path) except OSError: # dir already exists, may happen when testing EDVerbose.screen( 'Target directory on pyarch ({0}) already exists, ignoring' .format(pyarch_path)) file_list = [] # we can now copy the files to this dir for f in os.listdir(original_files_dir): current = os.path.join(original_files_dir, f) if not os.path.isfile(current): continue if not os.path.splitext( current)[1].lower() in ISPYB_UPLOAD_EXTENSIONS: continue new_path = os.path.join(pyarch_path, f) file_list.append(new_path) shutil.copyfile(current, new_path) # now add those to the ispyb upload for path in file_list: dirname, filename = os.path.split(path) attach = AutoProcProgramAttachment() attach.fileType = "Result" attach.fileName = filename attach.filePath = dirname program_container.AutoProcProgramAttachment.append(attach) program_container.AutoProcProgram.processingStatus = True output.AutoProcProgramContainer = program_container # first with anom output.AutoProcScalingContainer = scaling_container_anom ispyb_input = XSDataInputStoreAutoProc() ispyb_input.AutoProcContainer = output with open(self.dataInput.output_file.path.value, 'w') as f: f.write(ispyb_input.marshal()) # store results in ispyb self.store_autoproc_anom.dataInput = ispyb_input t0 = time.time() self.store_autoproc_anom.executeSynchronous() self.stats['ispyb_upload'] = time.time() - t0 with open(self.log_file_path, 'w') as f: json.dump(self.stats, f) if self.store_autoproc_anom.isFailure(): self.ERROR('could not send results to ispyb') else: # store the autoproc ID as a filename in the # fastproc_integration_ids directory os.mknod( os.path.join(self.autoproc_ids_dir, str(self.integration_id_anom)), 0755) # then noanom stats output.AutoProcScalingContainer = scaling_container_noanom ispyb_input = XSDataInputStoreAutoProc() ispyb_input.AutoProcContainer = output with open(self.dataInput.output_file.path.value, 'w') as f: f.write(ispyb_input.marshal()) # store results in ispyb self.store_autoproc_noanom.dataInput = ispyb_input t0 = time.time() self.store_autoproc_noanom.executeSynchronous() self.stats['ispyb_upload'] = time.time() - t0 with open(self.log_file_path, 'w') as f: json.dump(self.stats, f) if self.store_autoproc_noanom.isFailure(): self.ERROR('could not send results to ispyb') else: # store the autoproc id os.mknod( os.path.join(self.autoproc_ids_dir, str(self.integration_id_noanom)), 0755)
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, isStaraniso, proposal, timeStart, timeEnd): if isAnom: anomString = "anom" else: anomString = "noanom" if isStaraniso: staranisoString = "_staraniso" else: staranisoString = "" # Read the generated ISPyB xml file pathToISPyBXML = None if isStaraniso: if edPluginExecAutoPROC.dataOutput.ispybXML_staraniso is not None: pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML_staraniso.path.value elif edPluginExecAutoPROC.dataOutput.ispybXML is not None: pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML.path.value if pathToISPyBXML is not None: autoProcContainer = AutoProcContainer.parseFile(pathToISPyBXML) # "Fix" certain entries in the ISPyB xml file autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics: if isAnom: autoProcScalingStatistics.anomalous = True else: autoProcScalingStatistics.anomalous = False # Convert from fraction to % autoProcScalingStatistics.rMerge *= 100.0 autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer image = autoProcIntegrationContainer.Image if self.dataInput.dataCollectionId is not None: image.dataCollectionId = self.dataInput.dataCollectionId.value autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer autoProcProgram = autoProcProgramContainer.AutoProcProgram if isAnom: autoProcIntegration.anomalous = True if isStaraniso: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnomStaraniso autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnomStaraniso else: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnom autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnom else: autoProcIntegration.anomalous = False if isStaraniso: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanomStaraniso autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanomStaraniso else: autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanom autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanom autoProcProgram.processingPrograms = "autoPROC" + staranisoString autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart) autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd) autoProcProgram.processingStatus = "SUCCESS" # EDNA-245 - remove "truncate_{early,late}-unique.mtz" from autoProcProgramContainer.AutoProcProgramAttachment autoProcProgramContainer.AutoProcProgramAttachment[:] = [x for x in autoProcProgramContainer.AutoProcProgramAttachment if not self.matchesTruncateEarlyLate(x.fileName) ] for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment: if autoProcProgramAttachment.fileName == "summary.html": # Check if summary_inlined.html exists summaryInlinedHtmlPath = os.path.join(autoProcProgramAttachment.filePath, "summary_inlined.html") if os.path.exists(summaryInlinedHtmlPath): summaryName = "summary_inlined" summaryHtmlPath = summaryInlinedHtmlPath else: summaryName = "summary" summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) # Replace opidXX with user name htmlSummary = open(summaryHtmlPath).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) htmlSummary = htmlSummary.replace(userString1, userString2) open(summaryHtmlPath, "w").write(htmlSummary) # Upload summary.html pathtoFile = summaryHtmlPath pyarchFile = self.pyarchPrefix + "_{0}_{1}.html".format(anomString, summaryName) if not pyarchFile in self.listPyarchFile: shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) self.listPyarchFile.append(pyarchFile) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = os.path.basename(pyarchFile) autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" if summaryName == "summary": # Convert the summary.html to summary.pdf xsDataInputHTML2PDF = XSDataInputHTML2PDF() xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath))) xsDataInputHTML2PDF.paperSize = XSDataString("A3") xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True) edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString)) edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF edPluginHTML2Pdf.executeSynchronous() pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile) # Copy file to results directory and pyarch shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile)) if self.pyarchDirectory is not None: shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile)) autoProcProgramAttachmentPdf = AutoProcProgramAttachment() autoProcProgramAttachmentPdf.fileName = pyarchPdfFile autoProcProgramAttachmentPdf.filePath = self.pyarchDirectory autoProcProgramAttachmentPdf.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachmentPdf) elif autoProcProgramAttachment.fileName == "truncate-unique.mtz": pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString) shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory else: pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName) pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile)) if self.pyarchDirectory is not None: shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile)) autoProcProgramAttachment.fileName = pyarchFile autoProcProgramAttachment.filePath = self.pyarchDirectory # Add XSCALE.LP file if present processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value pathToXSCALELog = os.path.join(processDirectory, "XSCALE.LP") if os.path.exists(pathToXSCALELog): pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString) shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog)) if self.pyarchDirectory is not None: shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXSCALELog autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add XDS_ASCII.HKL if present and gzip it pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL") if os.path.exists(pathToXdsAsciiHkl) and self.pyarchDirectory is not None: pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString) f_in = open(pathToXdsAsciiHkl) f_out = gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), "wb") f_out.writelines(f_in) f_out.close() f_in.close() shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Result" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add log file pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value autoPROClog = open(pathToLogFile).read() userString1 = "User : {0} (".format(os.environ["USER"]) userString2 = "User : {0} (".format(proposal) autoPROClog = autoPROClog.replace(userString1, userString2) open(pathToLogFile, "w").write(autoPROClog) pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString) shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile)) if self.pyarchDirectory is not None: shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchLogFile autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Add report.pdf pathToRepordPdf = None if isStaraniso and edPluginExecAutoPROC.dataOutput.reportPdf_staraniso is not None: pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf_staraniso.path.value elif edPluginExecAutoPROC.dataOutput.reportPdf is not None: pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf.path.value if pathToRepordPdf is not None: pyarchReportFile = self.pyarchPrefix + "_{0}_{1}".format(anomString, os.path.basename(pathToRepordPdf)) shutil.copy(pathToRepordPdf, os.path.join(self.resultsDirectory, pyarchReportFile)) if self.pyarchDirectory is not None: shutil.copy(pathToRepordPdf, os.path.join(self.pyarchDirectory, pyarchReportFile)) autoProcProgramAttachment = AutoProcProgramAttachment() autoProcProgramAttachment.fileName = pyarchReportFile autoProcProgramAttachment.filePath = self.pyarchDirectory autoProcProgramAttachment.fileType = "Log" autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment) # Upload the xml to ISPyB xsDataInputStoreAutoProc = XSDataInputStoreAutoProc() xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer edPluginStoreAutoprocAnom = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "ISPyBStoreAutoProcv1_4_{0}{1}".format(anomString, staranisoString)) edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc edPluginStoreAutoprocAnom.executeSynchronous() isSuccess = not edPluginStoreAutoprocAnom.isFailure() if isSuccess: self.screen("{0}{1} results uploaded to ISPyB".format(anomString, staranisoString)) if isAnom: if isStaraniso: self.hasUploadedAnomStaranisoResultsToISPyB = True else: self.hasUploadedAnomResultsToISPyB = True else: if isStaraniso: self.hasUploadedNoanomStaranisoResultsToISPyB = True else: self.hasUploadedNoanomResultsToISPyB = True else: self.screen("Could not upload {0}{1} results to ISPyB".format(anomString, staranisoString))